diff --git a/regtest.py b/regtest.py index 6794648..bb80f4c 100755 --- a/regtest.py +++ b/regtest.py @@ -26,6 +26,7 @@ import test_util import test_report as report import test_coverage as coverage +import test_asv safe_flags = ['TEST', 'USE_CUDA', 'USE_ACC', 'USE_MPI', 'USE_OMP', 'DEBUG', 'USE_GPU'] @@ -357,11 +358,9 @@ def test_suite(argv): #-------------------------------------------------------------------------- # check bench dir and create output directories #-------------------------------------------------------------------------- + bench_dir = suite.get_bench_dir() all_compile = all([t.compileTest == 1 for t in test_list]) - if not all_compile: - bench_dir = suite.get_bench_dir() - if not args.copy_benchmarks is None: last_run = suite.get_last_run() @@ -522,6 +521,10 @@ def test_suite(argv): os.chdir(bdir) + comp_string = "" + executable = None + rc = 1 + if test.reClean == 1: # for one reason or another, multiple tests use different # build options, make clean again to be safe @@ -613,6 +616,8 @@ def test_suite(argv): act = shutil.move else: suite.log.fail("invalid action") + skip_to_next_test = 1 + break try: act(nfile, output_dir) @@ -1265,6 +1270,11 @@ def test_suite(argv): with open(file_path, 'w') as json_file: json.dump(runtimes, json_file, indent=4) + #-------------------------------------------------------------------------- + # output ASV results + #-------------------------------------------------------------------------- + test_asv.save_asv_history(suite, test_list) + #-------------------------------------------------------------------------- # parameter coverage #-------------------------------------------------------------------------- diff --git a/suite.py b/suite.py index bebe5cf..3f7f250 100644 --- a/suite.py +++ b/suite.py @@ -8,6 +8,11 @@ import test_util import tempfile as tf +plt = None +dates = None +HAVE_MPL = False +HAVE_BOKEH = False + try: from json.decoder import JSONDecodeError except ImportError: JSONDecodeError = ValueError @@ -19,6 +24,7 @@ from bokeh.resources import CDN from bokeh.models import HoverTool from datetime import datetime as dt + HAVE_BOKEH = True except: try: @@ -28,11 +34,13 @@ else: matplotlib.use('Agg') import matplotlib.pyplot as plt + HAVE_MPL = True try: import matplotlib.dates as dates except: DO_TIMINGS_PLOTS = False + HAVE_MPL = False class Test: @@ -787,18 +795,19 @@ def make_timing_plots(self, active_test_list=None, valid_dirs=None, all_tests=No valid_dirs, all_tests = self.get_run_history(active_test_list) timings = self.get_wallclock_history() - try: bokeh - except NameError: + if not DO_TIMINGS_PLOTS: + return + if HAVE_BOKEH: + convf = lambda s: dt.strptime(s, '%Y-%m-%d') + using_mpl = False + self.plot_ext = "html" + elif HAVE_MPL and dates is not None: convf = dates.datestr2num using_mpl = True self.plot_ext = "png" - else: - - convf = lambda s: dt.strptime(s, '%Y-%m-%d') - using_mpl = False - self.plot_ext = "html" + return def convert_date(date): """ Convert to a matplotlib readable date""" @@ -1087,9 +1096,13 @@ def build_tools(self, test_list): if ("DiffSameDomainRefined3d" in self.extra_tools): extra_tools.append("DiffSameDomainRefined3d") for t in extra_tools: + ndim = None if ("1d" in t): ndim=1 - if ("2d" in t): ndim=2 - if ("3d" in t): ndim=3 + elif ("2d" in t): ndim=2 + elif ("3d" in t): ndim=3 + else: + self.log.fail(f"unable to determine dimension for tool {t}") + continue self.log.log(f"building {t}...") comp_string, rc = self.build_c(opts= f"EBASE=DiffSameDomainRefined DIM={ndim} DEBUG=FALSE USE_MPI=FALSE USE_OMP=FALSE ") diff --git a/test_asv.py b/test_asv.py new file mode 100644 index 0000000..9e2e27d --- /dev/null +++ b/test_asv.py @@ -0,0 +1,89 @@ +import json +import os +import platform +import time + +def save_asv_history(suite, test_list): + """ + Save test results in Airspeed Velocity (ASV) format. + """ + + # Get the commit hash from the source repo + commit_hash = "unknown" + if suite.sourceTree in suite.repos: + if suite.repos[suite.sourceTree].hash_current: + commit_hash = suite.repos[suite.sourceTree].hash_current.strip() + + if commit_hash == "unknown": + # Fallback: use the first repo or "source" + if "source" in suite.repos and suite.repos["source"].hash_current: + commit_hash = suite.repos["source"].hash_current.strip() + elif len(suite.repos) > 0: + # Use the first available repo hash + first_repo = list(suite.repos.values())[0] + if first_repo.hash_current: + commit_hash = first_repo.hash_current.strip() + + # Current timestamp in milliseconds + timestamp = int(time.time() * 1000) + + # Environment and Machine Info + machine_name = platform.node() + os_name = platform.system() + python_version = platform.python_version() + + params = { + "machine": machine_name, + "os": os_name, + "python": python_version + } + + results = {} + for test in test_list: + # We only care about tests that passed execution and have a wall time + # Use record_runtime logic but we can be slightly more permissive if needed, + # but keeping it consistent with wallclock_history is good. + if test.record_runtime(suite): + results[test.name] = test.wall_time + + if not results: + return + + data = { + "version": 1, + "commit_hash": commit_hash, + "env_name": f"regtest-{machine_name}", + "date": timestamp, + "params": params, + "results": results, + "requirements": {} + } + + # Determine output path + # We'll put it in a 'asv' subdirectory within the benchmarks directory + try: + bench_dir = suite.get_bench_dir() + except SystemExit: + # If get_bench_dir fails (e.g. dir doesn't exist and strict checking), + # we might skip ASV saving or try to create it if allowed. + # But regtest.py usually ensures bench_dir is checked. + return + + asv_dir = os.path.join(bench_dir, "asv") + machine_dir = os.path.join(asv_dir, machine_name) + + if not os.path.exists(machine_dir): + try: + os.makedirs(machine_dir) + except OSError: + suite.log.warn(f"Could not create ASV directory: {machine_dir}") + return + + output_file = os.path.join(machine_dir, f"{commit_hash}.json") + + try: + with open(output_file, 'w') as f: + json.dump(data, f, indent=4) + suite.log.log(f"ASV results saved to {output_file}") + except Exception as e: + suite.log.warn(f"Failed to save ASV results: {e}") diff --git a/test_report.py b/test_report.py index 9c1bdea..885dc69 100644 --- a/test_report.py +++ b/test_report.py @@ -333,6 +333,9 @@ def report_single_test(suite, test, tests, failure_msg=None): # we stored compilation success in the test object compile_successful = test.compile_successful + compare_successful = False + diff_lines = [] + analysis_successful = True if test.analysisRoutine != '': analysis_successful = test.analysis_successful @@ -846,6 +849,7 @@ def report_this_test_run(suite, make_benchmarks, note, update_time, status_file = "%s.status" % (test.name) status = None + td_class = "unknown" with open(status_file) as sf: for line in sf: if line.find("PASSED") >= 0: