Merge pull request #156 from faster-cpython/color-results

Highlight interpreter heavy benchmarks

Author: mdboom

Diff for: bench_runner/plot.py

@@ -21,21 +21,49 @@
 from . import result
 
 
+INTERPRETER_HEAVY = {
+    "chaos",
+    "coroutines",
+    "deepcopy",
+    "deltablue",
+    "generators",
+    "go",
+    "hexiom",
+    "logging",
+    "nbody",
+    "pickle_pure_python",
+    "pprint",
+    "raytrace",
+    "richards",
+    "richards_super",
+    "sqlglot_parse",
+    "tomli_loads",
+    "unpack_sequence",
+    "unpickle_pure_python",
+}
+
+
 def plot_diff_pair(ax, data):
     if not len(data):
         return []
 
     all_data = []
     violins = []
+    colors = []
 
     for i, (name, values, mean) in enumerate(data):
         if values is not None:
             idx = np.round(np.linspace(0, len(values) - 1, 100)).astype(int)
             violins.append(values[idx])
             all_data.extend(values)
+            if name in INTERPRETER_HEAVY:
+                colors.append("red")
+            else:
+                colors.append("C0")
         else:
             violins.append([1.0])
             all_data.extend([1.0])
+            colors.append("C0")
             ax.text(1.01, i + 1, "insignificant")
 
     violins.append(all_data)
@@ -50,6 +78,8 @@ def plot_diff_pair(ax, data):
     )
 
     violin["cquantiles"].set_linestyle(":")
+    for body, color in zip(violin["bodies"], colors):
+        body.set_facecolor(color)
 
     for i, values in enumerate(violins):
         if not np.all(values == [1.0]):

Diff for: bench_runner/scripts/generate_results.py

@@ -5,6 +5,7 @@
 from collections import defaultdict
 import datetime
 import io
+import multiprocessing
 from pathlib import Path
 import sys
 from typing import Iterable, Optional, TextIO
@@ -133,27 +134,32 @@ def write_pystats_diff(filename: Path, compare: Comparison) -> None:
 }
 
 
+def _worker(args):
+    func, output_filename, compare = args
+    func(output_filename, compare)
+
+
 def save_generated_results(results: Iterable[Result], force: bool = False) -> None:
     """
     Write out the comparison tables and plots for every result.
 
     By default, files are only written out if they don't already exist. To force
     regeneration, pass ``force=True``.
     """
+    work = []
     for result in results:
         for compare in result.bases.values():
             if compare.filename is not None:
                 for suffix, func in RESULT_TYPES[result.result_info[0]].items():
                     filename = compare.filename.parent / (
                         compare.filename.stem + suffix
                     )
-                    if not filename.exists() or force:
-                        util.status(".")
-                        func(filename, compare)
-                    else:
-                        util.status("/")
+                    if not filename.exists() or force:  # or suffix == ".png":
+                        work.append((func, filename, compare))
 
-    print()
+    pool = multiprocessing.Pool()
+    for i, _ in enumerate(pool.imap_unordered(_worker, work)):
+        print(f"{i:04d}/{len(work):04d}", end="\r")
 
 
 def output_results_index(