Merge branch 'evaluation' into 'dev'

Evaluation for comparison of results from regression between images for synthesis

See merge request !44

Author: wyli

niftynet/evaluation/compare_regressions.py

@@ -0,0 +1,69 @@
+from __future__ import absolute_import, print_function
+
+import os.path
+
+import nibabel as nib
+import numpy as np
+
+import niftynet.utilities.csv_table as csv_table
+from niftynet.evaluation.pairwise_measures import PairwiseMeasuresRegression
+
+MEASURES = (
+    'mse','rmse','mae','r2'
+)
+# MEASURES_NEW = ('ref volume', 'reg volume', 'tp', 'fp', 'fn', 'outline_error',
+#             'detection_error', 'dice')
+OUTPUT_FORMAT = '{:4f}'
+OUTPUT_FILE_PREFIX = 'PairwiseMeasureReg'
+
+
+def run(param, csv_dict):
+    # output
+    out_name = '{}_{}_{}.csv'.format(
+        OUTPUT_FILE_PREFIX,
+        os.path.split(param.ref_dir)[1],
+        os.path.split(param.seg_dir)[1])
+    print("Writing {} to {}".format(out_name, param.save_csv_dir))
+
+    # inputs
+    csv_loader = csv_table.CSVTable(csv_dict=csv_dict, allow_missing=False)
+    reg_names = [csv_loader._csv_table[m][1][0][0] for m in range(
+        0, len(csv_loader._csv_table))]
+    ref_names = [csv_loader._csv_table[m][2][0][0] for m in range(
+        0, len(csv_loader._csv_table))]
+    # reg_names = util.list_files(param.reg_dir, param.ext)
+    # ref_names = util.list_files(param.ref_dir, param.ext)
+    pair_list = list(zip(reg_names, ref_names))
+    # TODO check reg_names ref_names matching
+    # TODO do we evaluate all combinations?
+    # import itertools
+    # pair_list = list(itertools.product(reg_names, ref_names))
+    print("List of references is {}".format(ref_names))
+    print("List of regressions is {}".format(reg_names))
+
+    # prepare a header for csv
+    with open(os.path.join(param.save_csv_dir, out_name), 'w+') as out_stream:
+        # a trivial PairwiseMeasures obj to produce header_str
+        m_headers = PairwiseMeasuresRegression(0, 0, measures=MEASURES).header_str()
+        out_stream.write("Name (ref), Name (reg)" + m_headers + '\n')
+
+        # do the pairwise evaluations
+        for i, pair_ in enumerate(pair_list):
+            reg_name = pair_[0]
+            ref_name = pair_[1]
+            print('>>> {} of {} evaluations, comparing {} and {}.'.format(
+                i + 1, len(pair_list), ref_name, reg_name))
+            reg_nii = nib.load(os.path.join(param.seg_dir, reg_name))
+            ref_nii = nib.load(os.path.join(param.ref_dir, ref_name))
+            voxel_sizes = reg_nii.header.get_zooms()[0:3]
+            reg = np.squeeze(reg_nii.get_data())
+            ref = np.squeeze(ref_nii.get_data())
+            assert (np.all(reg) >= 0)
+            assert (np.all(ref) >= 0)
+            assert (reg.shape == ref.shape)
+            PE = PairwiseMeasuresRegression(reg, ref,
+                                  measures=MEASURES)
+            fixed_fields = "{}, {}, ".format(ref_name, reg_name)
+            out_stream.write(fixed_fields + PE.to_string(
+                OUTPUT_FORMAT) + '\n')
+

niftynet/evaluation/pairwise_measures.py

@@ -264,3 +264,53 @@ def to_string(self, fmt='{:.4f}'):
                 if isinstance(result, tuple) else fmt.format(result)
             result_str += ','
         return result_str[:-1]  # trim the last comma
+
+
+class PairwiseMeasuresRegression(object):
+    def __init__(self, reg_img, ref_img, measures=None):
+
+        self.reg = reg_img
+        self.ref = ref_img
+        self.measures = measures
+
+        self.m_dict = {
+            'mse': (self.mse, 'MSE'),
+            'rmse': (self.rmse, 'RMSE'),
+            'mae': (self.mae, 'MAE'),
+            'r2': (self.r2, 'R2')
+        }
+
+
+    def mse(self):
+        return np.mean(np.square(self.reg - self.ref))
+
+
+    def rmse(self):
+        return np.sqrt(self.mse())
+
+
+    def mae(self):
+        return np.mean(np.abs(self.ref-self.reg))
+
+    def r2(self):
+        ref_var = np.sum(np.square(self.ref-np.mean(self.ref)))
+        reg_var = np.sum(np.square(self.reg-np.mean(self.reg)))
+        cov_refreg = np.sum((self.reg-np.mean(self.reg))*(self.ref-np.mean(
+            self.ref)))
+        return np.square(cov_refreg / np.sqrt(ref_var*reg_var+0.00001))
+
+
+    def header_str(self):
+        result_str = [self.m_dict[key][1] for key in self.measures]
+        result_str = ',' + ','.join(result_str)
+        return result_str
+
+
+    def to_string(self, fmt='{:.4f}'):
+        result_str = ""
+        for key in self.measures:
+            result = self.m_dict[key][0]()
+            result_str += ','.join(fmt.format(x) for x in result) \
+                if isinstance(result, tuple) else fmt.format(result)
+            result_str += ','
+        return result_str[:-1]  # trim the last comma

niftynet/utilities/misc_csv.py

@@ -112,14 +112,15 @@ def __find_max_overlap_in_list(name, list_names):
     if len(list_names) == 0:
         return '', -1
     for test in list_names:
-        match = SequenceMatcher(None, name, test).find_longest_match(
-            0, len(name), 0, len(test))
-        if match.size >= match_max and match.size/len(test) >= \
-                match_ratio:
-            match_max = match.size
-            match_seq = test[match.b:(match.b + match.size)]
-            match_ratio = match.size/len(test)
-            match_orig = test
+        if len(test)>0:
+            match = SequenceMatcher(None, name, test).find_longest_match(
+                0, len(name), 0, len(test))
+            if match.size >= match_max and match.size/len(test) >= \
+                    match_ratio:
+                match_max = match.size
+                match_seq = test[match.b:(match.b + match.size)]
+                match_ratio = match.size/len(test)
+                match_orig = test
     if match_max == 0:
         return '', -1
     other_list = [name for name in list_names if match_seq in name and

run_evaluation.py

@@ -13,6 +13,10 @@
 
         niftynet.evaluation.compute_ROI_statistics.run(args, csv_dict)
     elif args.action.lower() == 'compare':
-        import niftynet.evaluation.compare_segmentations
+        if args.application_type == 'segmentation':
+            import niftynet.evaluation.compare_segmentations
 
-        niftynet.evaluation.compare_segmentations.run(args, csv_dict)
+            niftynet.evaluation.compare_segmentations.run(args, csv_dict)
+        elif args.application_type == 'regression':
+            import niftynet.evaluation.compare_regressions
+            niftynet.evaluation.compare_regressions.run(args,csv_dict)