add airspeed velocity-compatible JSON output

regtest.py

@@ -26,6 +26,7 @@
 import test_util
 import test_report as report
 import test_coverage as coverage
+import test_asv
 
 safe_flags = ['TEST', 'USE_CUDA', 'USE_ACC', 'USE_MPI', 'USE_OMP', 'DEBUG', 'USE_GPU']
 
@@ -357,11 +358,9 @@ def test_suite(argv):
     #--------------------------------------------------------------------------
     # check bench dir and create output directories
     #--------------------------------------------------------------------------
+    bench_dir = suite.get_bench_dir()
     all_compile = all([t.compileTest == 1 for t in test_list])
 
-    if not all_compile:
-        bench_dir = suite.get_bench_dir()
-
     if not args.copy_benchmarks is None:
         last_run = suite.get_last_run()
 
@@ -522,6 +521,10 @@ def test_suite(argv):
 
         os.chdir(bdir)
 
+        comp_string = ""
+        executable = None
+        rc = 1
+
         if test.reClean == 1:
             # for one reason or another, multiple tests use different
             # build options, make clean again to be safe
@@ -613,6 +616,8 @@ def test_suite(argv):
                 act = shutil.move
             else:
                 suite.log.fail("invalid action")
+                skip_to_next_test = 1
+                break
 
             try:
                 act(nfile, output_dir)
@@ -1265,6 +1270,11 @@ def test_suite(argv):
     with open(file_path, 'w') as json_file:
         json.dump(runtimes, json_file, indent=4)
 
+    #--------------------------------------------------------------------------
+    # output ASV results
+    #--------------------------------------------------------------------------
+    test_asv.save_asv_history(suite, test_list)
+
     #--------------------------------------------------------------------------
     # parameter coverage
     #--------------------------------------------------------------------------

suite.py

@@ -8,6 +8,11 @@
 import test_util
 import tempfile as tf
 
+plt = None
+dates = None
+HAVE_MPL = False
+HAVE_BOKEH = False
+
 try: from json.decoder import JSONDecodeError
 except ImportError: JSONDecodeError = ValueError
 
@@ -19,6 +24,7 @@
     from bokeh.resources import CDN
     from bokeh.models import HoverTool
     from datetime import datetime as dt
+    HAVE_BOKEH = True
 
 except:
     try:
@@ -28,11 +34,13 @@
     else:
         matplotlib.use('Agg')
         import matplotlib.pyplot as plt
+        HAVE_MPL = True
 
     try:
         import matplotlib.dates as dates
     except:
         DO_TIMINGS_PLOTS = False
+        HAVE_MPL = False
 
 class Test:
 
@@ -787,18 +795,19 @@ def make_timing_plots(self, active_test_list=None, valid_dirs=None, all_tests=No
             valid_dirs, all_tests = self.get_run_history(active_test_list)
         timings = self.get_wallclock_history()
 
-        try: bokeh
-        except NameError:
+        if not DO_TIMINGS_PLOTS:
+            return
 
+        if HAVE_BOKEH:
+            convf = lambda s: dt.strptime(s, '%Y-%m-%d')
+            using_mpl = False
+            self.plot_ext = "html"
+        elif HAVE_MPL and dates is not None:
             convf = dates.datestr2num
             using_mpl = True
             self.plot_ext = "png"
-
         else:
-
-            convf = lambda s: dt.strptime(s, '%Y-%m-%d')
-            using_mpl = False
-            self.plot_ext = "html"
+            return
 
         def convert_date(date):
             """ Convert to a matplotlib readable date"""
@@ -1087,9 +1096,13 @@ def build_tools(self, test_list):
             if ("DiffSameDomainRefined3d" in self.extra_tools): extra_tools.append("DiffSameDomainRefined3d")
 
             for t in extra_tools:
+                ndim = None
                 if ("1d" in t): ndim=1
-                if ("2d" in t): ndim=2
-                if ("3d" in t): ndim=3
+                elif ("2d" in t): ndim=2
+                elif ("3d" in t): ndim=3
+                else:
+                    self.log.fail(f"unable to determine dimension for tool {t}")
+                    continue
                 self.log.log(f"building {t}...")
                 comp_string, rc = self.build_c(opts=
                         f"EBASE=DiffSameDomainRefined DIM={ndim} DEBUG=FALSE USE_MPI=FALSE USE_OMP=FALSE ")

test_asv.py

@@ -0,0 +1,89 @@
+import json
+import os
+import platform
+import time
+
+def save_asv_history(suite, test_list):
+    """
+    Save test results in Airspeed Velocity (ASV) format.
+    """
+
+    # Get the commit hash from the source repo
+    commit_hash = "unknown"
+    if suite.sourceTree in suite.repos:
+        if suite.repos[suite.sourceTree].hash_current:
+             commit_hash = suite.repos[suite.sourceTree].hash_current.strip()
+
+    if commit_hash == "unknown":
+        # Fallback: use the first repo or "source"
+        if "source" in suite.repos and suite.repos["source"].hash_current:
+            commit_hash = suite.repos["source"].hash_current.strip()
+        elif len(suite.repos) > 0:
+            # Use the first available repo hash
+            first_repo = list(suite.repos.values())[0]
+            if first_repo.hash_current:
+                commit_hash = first_repo.hash_current.strip()
+
+    # Current timestamp in milliseconds
+    timestamp = int(time.time() * 1000)
+
+    # Environment and Machine Info
+    machine_name = platform.node()
+    os_name = platform.system()
+    python_version = platform.python_version()
+
+    params = {
+        "machine": machine_name,
+        "os": os_name,
+        "python": python_version
+    }
+
+    results = {}
+    for test in test_list:
+        # We only care about tests that passed execution and have a wall time
+        # Use record_runtime logic but we can be slightly more permissive if needed,
+        # but keeping it consistent with wallclock_history is good.
+        if test.record_runtime(suite):
+             results[test.name] = test.wall_time
+
+    if not results:
+        return
+
+    data = {
+        "version": 1,
+        "commit_hash": commit_hash,
+        "env_name": f"regtest-{machine_name}",
+        "date": timestamp,
+        "params": params,
+        "results": results,
+        "requirements": {} 
+    }
+
+    # Determine output path
+    # We'll put it in a 'asv' subdirectory within the benchmarks directory
+    try:
+        bench_dir = suite.get_bench_dir()
+    except SystemExit:
+        # If get_bench_dir fails (e.g. dir doesn't exist and strict checking), 
+        # we might skip ASV saving or try to create it if allowed.
+        # But regtest.py usually ensures bench_dir is checked.
+        return
+
+    asv_dir = os.path.join(bench_dir, "asv")
+    machine_dir = os.path.join(asv_dir, machine_name)
+
+    if not os.path.exists(machine_dir):
+        try:
+            os.makedirs(machine_dir)
+        except OSError:
+            suite.log.warn(f"Could not create ASV directory: {machine_dir}")
+            return
+
+    output_file = os.path.join(machine_dir, f"{commit_hash}.json")
+
+    try:
+        with open(output_file, 'w') as f:
+            json.dump(data, f, indent=4)
+        suite.log.log(f"ASV results saved to {output_file}")
+    except Exception as e:
+        suite.log.warn(f"Failed to save ASV results: {e}")

test_report.py

@@ -333,6 +333,9 @@ def report_single_test(suite, test, tests, failure_msg=None):
     # we stored compilation success in the test object
     compile_successful = test.compile_successful
 
+    compare_successful = False
+    diff_lines = []
+
     analysis_successful = True
     if test.analysisRoutine != '':
         analysis_successful = test.analysis_successful
@@ -846,6 +849,7 @@ def report_this_test_run(suite, make_benchmarks, note, update_time,
             status_file = "%s.status" % (test.name)
 
             status = None
+            td_class = "unknown"
             with open(status_file) as sf:
                 for line in sf:
                     if line.find("PASSED") >= 0: