Merge pull request #2661 from jsiirola/compare-update

Improve compare.py handling of nosetests/pytest output

Author: jsiirola

scripts/performance/compare.py

@@ -136,17 +136,56 @@ def combine(*results):
     for result in results:
         for dataset in result[1:]:
             for test, result in dataset.items():
+                if "::" not in test:
+                    # Convert nosetests results in to pytest format
+                    path, test = test.split(':')
+                    test = '/'.join(path.split('.')) + '.py::' \
+                           + '::'.join(test.split('.'))
                 if test not in ans:
                     ans[test] = {}
                 testdata = ans[test]
                 for metric, value in result.items():
                     if type(value) is dict:
                         continue
                     testdata.setdefault(metric, []).append(value)
+    # Nosetests and pytest would give different test names (based on
+    # where they started including path elements).  We will assume that
+    # tests should be uniquely declared by the test file, class, and
+    # test name.  So, any two tests where one name ends with the
+    # complete name of the other will be assumed to be the same test and
+    # combined.
+    for base in list(ans):
+        for test in ans:
+            if test == base:
+                break
+            if test.endswith(base):
+                testdata = ans[test]
+                otherdata = ans.pop(base)
+                for metric, value in otherdata.items():
+                    testdata.setdefault(metric, []).append(value)
+                break
     return ans
 
 def compare(base_data, test_data):
     """Compare two data sets (generated by compare())"""
+    # Nosetests and pytest would give different test names (based on
+    # where they started including path elements).  We will assume that
+    # tests should be uniquely declared by the test file, class, and
+    # test name.  So, any two tests where one name ends with the
+    # complete name of the other will be assumed to be the same test.
+    # We will make both to the "more specific" (longer) name before
+    # comparing.
+    for base in list(base_data):
+        for test in test_data:
+            if base == test:
+                break
+            if test.endswith(base):
+                base_data[test] = base_data.pop(base)
+                break
+            if base.endswith(test):
+                test_data[base] = test_data.pop(test)
+                break
+
     fields = set()
     for testname, base in base_data.items():
         if testname not in test_data: