version 0.0.9: added iou threshold to detection metrics

Author: nwoyecid

README.rst

@@ -204,6 +204,7 @@ The following function are possible with the ``Detection`` class:
 |                                   | corresponding groundtruth. Each   |
 |                                   | frame prediction/groundtruth can  |
 |                                   | be either as a ``list of list``   |
+|                                   | or as a ``list of dict``.         |
 |                                   | (more details below).             |
 +-----------------------------------+-----------------------------------+
 | video_end()                       | Call to make the end of one video |
@@ -236,7 +237,6 @@ The following function are possible with the ``Detection`` class:
 |                                   | for all seen samples.             |
 +-----------------------------------+-----------------------------------+
 
-.. _args-1:
 
 args:
 ~~~~~
@@ -261,7 +261,7 @@ args:
 4. ``component`` can be any of the following (‘i’, ‘v’, ‘t’, ‘iv’,
    ‘it’,‘ivt’) to compute performance for (instrument, verb, target,
    instrument-verb, instrument-target, instrument-verb-target)
-   respectively, default is ‘ivt’ for triplets.<
+   respectively, default is ‘ivt’ for triplets.
 
 -  the output is a ``dict`` with keys(“AP”, “mAP”, “Rec”, “mRec”, “Pre”,
    “mPre”) for per-class AP, mean AP, per-class Recall, mean Recall,
@@ -332,13 +332,14 @@ Although, the ``Detection()`` and ``Recognition()`` classes uses the ``Disentang
 Afterwards, any of the component's predictions/labels can be filtered from the main triplet's predictions/labels as follows:
 
 .. code:: python
+
    i_labels = filter.extract(inputs=ivt_labels, component="i")
    v_preds  = filter.extract(inputs=ivt_preds, component="v")
    t_preds  = filter.extract(inputs=ivt_preds, component="t")
    iv_labels = filter.extract(inputs=ivt_labels, component="iv")
    it_labels = filter.extract(inputs=ivt_labels, component="it")
 
-
+In this example, labels = (batch of) vector of groundtruth labels; preds = (batch of) vector of predicted probability scores.
 
 
 

ivtmetrics/__init__.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 __library__ = "ivtmetrics"
-__version__ = "0.0.8"
+__version__ = "0.0.9"
 __author__  = 'Chinedu Nwoye'
 __supervisor__  = 'Nicolas Padoy'
 __credits__ = 'CAMMA, ICube Lab, University of Strasbourg, France'

ivtmetrics/detection.py

@@ -9,6 +9,7 @@
 #%%%%%%%% imports %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 import numpy as np
 import sys
+import warnings
 from ivtmetrics.recognition import Recognition
 
 #%%%%%%%%%% RECOGNITION %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -50,7 +51,7 @@ class Detection(Recognition):
           add compute_global_AP('i/ivt') return AP for all seen examples
           add reset_video()
     """
-    def __init__(self, num_class=100, num_tool=6):
+    def __init__(self, num_class=100, num_tool=6, threshold=0.5):
         super(Recognition, self).__init__()
         self.num_class      = num_class  
         self.num_tool       = num_tool                
@@ -60,6 +61,7 @@ def __init__(self, num_class=100, num_tool=6):
         self.accumulator    = {}
         self.video_count    = 0
         self.end_call       = False
+        self.threshold      = threshold
         self.reset()        
 
     def reset(self):
@@ -115,6 +117,7 @@ def is_match(self, det_gt, det_pd, threshold):
         return status    
 
     def list2stack(self, x):
+        if x == []: x = [[-1,-1,-1,-1,-1,-1]] # empty
         #x format for a single frame: list(list): each list = [tripletID, toolID, toolProbs, x, y, w, h] bbox is scaled (0..1)
         assert isinstance(x[0], list), "Each frame must be a list of lists, each list a prediction of triplet and object locations"
         x = np.stack(x, axis=0)
@@ -136,7 +139,6 @@ def dict2stack(self, x):
             p = [d['triplet']]
             p.extend(d["instrument"])
             y.append(p)
-#        y = np.stack(y, axis=0)
         return self.list2stack(y)    
 
     def update(self, targets, predictions, format="list"): 
@@ -160,38 +162,38 @@ def update_frame(self, targets, predictions, format="list"):
             sys.exit("unkown input format for update function. Must be a list or dict")
         if len(detection_pd) + len(detection_gt) == 0:
             return
-        detection_gt_i = detection_gt.copy()
-        detection_pd_i = detection_pd.copy()
+        detection_gt_ivt = detection_gt.copy()
+        detection_pd_ivt = detection_pd.copy()
         # for triplet
-        for gt in detection_gt: 
+        for gt in detection_gt_ivt: 
             self.accumulator[self.video_count]["npos"][int(gt[0])] += 1
-        for det_pd in detection_pd:
+        for det_pd in detection_pd_ivt:
             self.accumulator[self.video_count]["ndet"][int(det_pd[0])] += 1
             matched = False
-            for k, det_gt in enumerate(detection_gt):
+            for k, det_gt in enumerate(detection_gt_ivt):
                 y = det_gt[0:] 
                 f = det_pd[0:]
-                if self.is_match(y, f, threshold=0.5):
-                    detection_gt = np.delete(detection_gt, obj=k, axis=0)
+                if self.is_match(y, f, threshold=self.threshold):
+                    detection_gt_ivt = np.delete(detection_gt_ivt, obj=k, axis=0)
                     matched = True
                     break
             if matched:
                 self.accumulator[self.video_count]["hits"][int(det_pd[0])].append(1.0)
             else:
                 self.accumulator[self.video_count]["hits"][int(det_pd[0])].append(0.0)
-        # for instrument
-        detection_gt = detection_gt_i
-        detection_pd = detection_pd_i
-        for gt in detection_gt:
+        # for instrument        
+        detection_gt_i = detection_gt.copy()
+        detection_pd_i = detection_pd.copy()
+        for gt in detection_gt_i:
             self.accumulator[self.video_count]["npos_i"][int(gt[1])] += 1
-        for det_pd in detection_pd:
+        for det_pd in detection_pd_i:
             self.accumulator[self.video_count]["ndet_i"][int(det_pd[1])] += 1
             matched = False
-            for k, det_gt in enumerate(detection_gt):                
-                y = det_gt[1:6] 
-                f = det_pd[1:6]
-                if self.is_match(y, f, threshold=0.5):
-                    detection_gt = np.delete(detection_gt, obj=k, axis=0)
+            for k, det_gt in enumerate(detection_gt_i):                
+                y = det_gt[1:] 
+                f = det_pd[1:]
+                if self.is_match(y, f, threshold=self.threshold):
+                    detection_gt_i = np.delete(detection_gt_i, obj=k, axis=0)
                     matched = True
                     break
             if matched:
@@ -240,7 +242,9 @@ def compute(self, component="ivt", video_id=None):
                 classwise_ap.append(ap)
                 classwise_rec.append(np.max(rec))
                 classwise_prec.append(np.max(prec))
-        return (classwise_ap, np.nanmean(classwise_ap)), (classwise_rec, np.nanmean(classwise_rec)), (classwise_prec, np.nanmean(classwise_prec))
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=RuntimeWarning)
+            return (classwise_ap, np.nanmean(classwise_ap)), (classwise_rec, np.nanmean(classwise_rec)), (classwise_prec, np.nanmean(classwise_prec))
 
     def compute_video_AP(self, component="ivt"):
         classwise_ap    = []
@@ -252,12 +256,14 @@ def compute_video_AP(self, component="ivt"):
             classwise_ap.append(ap)
             classwise_rec.append(rec)
             classwise_prec.append(prec)
-        classwise_ap    = np.nanmean(np.stack(classwise_ap, axis=0), axis=0)
-        classwise_rec   = np.nanmean(np.stack(classwise_rec, axis=0), axis=0)
-        classwise_prec  = np.nanmean(np.stack(classwise_prec, axis=0), axis=0)        
-        mAP             = np.nanmean(classwise_ap)
-        mRec            = np.nanmean(classwise_rec)
-        mPrec           = np.nanmean(classwise_prec) 
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=RuntimeWarning)
+            classwise_ap    = np.nanmean(np.stack(classwise_ap, axis=0), axis=0)
+            classwise_rec   = np.nanmean(np.stack(classwise_rec, axis=0), axis=0)
+            classwise_prec  = np.nanmean(np.stack(classwise_prec, axis=0), axis=0)        
+            mAP             = np.nanmean(classwise_ap)
+            mRec            = np.nanmean(classwise_rec)
+            mPrec           = np.nanmean(classwise_prec) 
         return {"AP":classwise_ap, "mAP":mAP, "Rec":classwise_rec, "mRec":mRec, "Pre":classwise_prec, "mPre":mPrec}     
 
     def compute_AP(self, component="ivt"):

ivtmetrics/disentangle.py

@@ -4,7 +4,7 @@
 An python implementation triplet component filtering .
 Created on Thu Dec 30 12:37:56 2021
 @author: nwoye chinedu i.
-icube, unistra
+(c) camma, icube, unistra
 """
 #%%%%%%%% imports %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 import numpy as np
@@ -77,7 +77,8 @@ def extract(self, inputs, component="i"):
             return np.array(list(map(self.decompose, inputs, component)))
 
     def map_file(self):
-        return np.array([ [ 0,  0,  2,  1,  2,  1],
+        return np.array([ 
+                       [ 0,  0,  2,  1,  2,  1],
                        [ 1,  0,  2,  0,  2,  0],
                        [ 2,  0,  2, 10,  2, 10],
                        [ 3,  0,  0,  3,  0,  3],
@@ -176,4 +177,5 @@ def map_file(self):
                        [96,  2,  9, 14, 29, 44],
                        [97,  3,  9, 14, 39, 59],
                        [98,  4,  9, 14, 49, 74],
-                       [99,  5,  9, 14, 59, 89]])
+                       [99,  5,  9, 14, 59, 89]
+                       ])

ivtmetrics/recognition.py

@@ -4,7 +4,7 @@
 An python implementation recognition AP for surgical action triplet evaluation.
 Created on Thu Dec 30 12:37:56 2021
 @author: nwoye chinedu i.
-icube, unistra
+(c) icube, unistra
 """
 #%%%%%%%% imports %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 import numpy as np
@@ -110,9 +110,9 @@ def compute_AP(self, component="ivt", ignore_null=False):
             targets  = self.extract(self.targets, component)
             predicts = self.extract(self.predictions, component)
         else:
-            sys.exit(f"Function filtering {component} not supported yet!")
+            sys.exit(f"Function filtering {component} not yet supported!")
         with warnings.catch_warnings():
-            warnings.filterwarnings(action='ignore', message='Mean of empty slice')            
+            warnings.filterwarnings(action='ignore', message='[info] triplet classes not represented in this test sample will be reported as nan values.')            
             classwise = average_precision_score(targets, predicts, average=None)
             if (ignore_null and component=="ivt"): classwise = classwise[:-6]
             mean      = np.nanmean(classwise)
@@ -143,9 +143,9 @@ def compute_global_AP(self, component="ivt", ignore_null=False):
             targets  = self.extract(targets, component)
             predicts = self.extract(predicts, component)
         else:
-            sys.exit(f"Function filtering {component} not supported yet!")            
+            sys.exit(f"Function filtering {component} not yet supported!")            
         with warnings.catch_warnings():
-            warnings.filterwarnings(action='ignore', message='Mean of empty slice')            
+            warnings.filterwarnings(action='ignore', message='[info] triplet classes not represented in this test sample will be reported as nan values.')            
             classwise = average_precision_score(targets, predicts, average=None)
             if (ignore_null and component=="ivt"): classwise = classwise[:-6]
             mean      = np.nanmean(classwise)
@@ -172,19 +172,19 @@ def compute_video_AP(self, component="ivt", ignore_null=False):
             global_predictions.append(self.predictions)
         video_log = []
         with warnings.catch_warnings():
-            warnings.filterwarnings(action='ignore', message='Mean of empty slice')
+            warnings.simplefilter("ignore", category=RuntimeWarning)
             for targets, predicts in zip(global_targets, global_predictions):
                 if component in ["ivt", "it", "iv", "t", "v", "i"]:
                     targets  = self.extract(targets, component)
                     predicts = self.extract(predicts, component)
                 else:
-                    sys.exit(f"Function filtering {component} not supported yet!")                        
+                    sys.exit(f"Function filtering {component} not yet supported!")                        
                 classwise = average_precision_score(targets, predicts, average=None)
                 video_log.append( classwise.reshape([1,-1]) )
-        video_log = np.concatenate(video_log, axis=0)         
-        videowise = np.nanmean(video_log, axis=0)
-        if (ignore_null and component=="ivt"): videowise = videowise[:-6]
-        mean      = np.nanmean(videowise)
+            video_log = np.concatenate(video_log, axis=0)         
+            videowise = np.nanmean(video_log, axis=0)
+            if (ignore_null and component=="ivt"): videowise = videowise[:-6]
+            mean      = np.nanmean(videowise)
         return {"AP":videowise, "mAP":mean}
 
     ##%%%%%%%%%%%%%%%%%%% TOP OP #%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%    

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name='ivtmetrics',
-    version='0.0.8',    
+    version='0.0.9',    
     packages=['ivtmetrics'],
     author='Chinedu Nwoye',
     author_email='nwoye.chinedu@gmail.com',    
@@ -15,7 +15,7 @@
     long_description = long_description,
     long_description_content_type ='text/x-rst',
     url='https://github.com/CAMMA-public/ivtmetrics',
-    download_url = 'https://github.com/CAMMA-public/ivtmetrics/archive/refs/tags/v0.0.8.tar.gz',    # I explain this later on
+    download_url = 'https://github.com/CAMMA-public/ivtmetrics/archive/refs/tags/v0.0.9.tar.gz',    # I explain this later on
     include_package_data=True,
     license='BSD 2-clause', # Chose a license from here: https://help.github.com/articles/licensing-a-repository
     install_requires=['scikit-learn',