Fix messy performance logic by moving file adjustment commands out of run_perf and into the ci script, no longer names directories as latest

Author: ldowen

.gitlab/performance.yml

@@ -13,11 +13,16 @@
     # Directory storing benchmark data is set in scripts/spheral_ats.py
     - INSTALL_DIR=$PERF_INSTALL_DIR/${SPEC//%/_}
     - cd $INSTALL_DIR
+    - INSTALL_BENCHMARK_DIR=$(./bin/spheral-ats --get-benchmark)
+    - PERF_OUT_DIR=$INSTALL_BENCHMARK_DIR/$(date +%s)
+    - mkdir PERF_OUT_DIR
     - echo "Running performance tests using install at $INSTALL_DIR"
-    - rm -rf perf_logs
-    - find tests/ -name '*.cali' -print -delete
+    - rm -rf perf_logs gpu_perf_logs
     - ./bin/spheral-ats --ciRun --numNodes 2 --logs perf_logs $INSTALL_DIR/$PERF_ATS_FILE
-    - ./bin/spheral-ats --gpu --ciRun --numNodes 2 --logs perf_logs $INSTALL_DIR/$PERF_ATS_FILE
+    - cp perf_logs/*.cali $PERF_OUT_DIR
+    - ./bin/spheral-ats --gpu --ciRun --numNodes 2 --logs gpu_perf_logs $INSTALL_DIR/$PERF_ATS_FILE
+    # Copy caliper files to the benchmark directory and don't complain if you cant
+    - cp gpu_perf_logs/*.cali $PERF_OUT_DIR 2>/dev/null || true
 
 toss_gcc_mpi_perf:
   extends: [.toss_perf_shell, .gcc_mpi, .run_perf]

.gitlab/scripts.yml

@@ -257,7 +257,7 @@ clean_old_perfs:
   # Prevent this job from getting artifacts
   dependencies: []
   script:
-    - BENCHMARK_LOC=$(python3 $SCRIPT_DIR/spheral_ats.py --get-benchmark)
+    - BENCHMARK_LOC=$(python3 $SCRIPT_DIR/spheral_ats.py --get-top-benchmark)
     - cd $BENCHMARK_LOC
     - find . -mindepth 2 -type d -mtime +200 -print -exec rm -rf {} +
 

scripts/devtools/performance_analysis.py

@@ -292,13 +292,14 @@ def main():
                     ref_files = curdata.metadata["benchmark_dir"].iloc[0]
                 except:
                     raise Exception("No reference or benchmark data specified")
-            # If we using benchmark reference data, only grab the current install/machine
+            # If using benchmark reference data, only grab the current install/machine
             # Get install config and machine name from current data
             install_config = curdata.metadata["install_config"].iloc[0]
             machine_name = curdata.metadata["cluster"].iloc[0]
-            ref_loc = os.path.join(ref_files, install_config, machine_name, "latest")
+            ref_loc = os.path.join(ref_files, install_config, machine_name)
             if (not os.path.exists(ref_loc)):
                 raise Exception(f"Benchmark location {ref_loc} does not exists")
+            ref_dir = sorted(glob.glob(os.path.join(ref_loc, "*")))[-1]
             cali_ref_files = glob.glob(os.path.join(ref_loc, "*.cali"), recursive=True)
 
     if (len(cali_ref_files) == 0):

scripts/lc/plot_hist_data.py

@@ -51,9 +51,10 @@ def extract_data(cali_file):
     return runday, rtimes
 
 def get_spec(cali_file):
-    # If cali_file is just a dir, grab a cali file from latest
+    # If cali_file is just a dir, grab the latest caliper file
     if os.path.isdir(cali_file):
-        cali_file = glob.glob(os.path.join(cali_file, "latest/*.cali"))[0]
+        perf_dir = sorted(glob.glob(os.path.join(cali_file, "*")))[-1]
+        cali_file = glob.glob(os.path.join(perf_dir, "*.cali"))[0]
     gls = cr.read_caliper_globals(cali_file)
     return gls["spec"]
 

scripts/spheral_ats.py

@@ -126,16 +126,20 @@ def main():
                         help="Set number of threads per rank to use. Currently only used by run_perf.py.")
     parser.add_argument("--batch", action="store_true", help="Submit job as batch.")
     parser.add_argument("--delay", action="store_true", help="Defer job until after 7 pm.")
-    parser.add_argument("--get-benchmark", action="store_true", help="Print benchmark location and stop.")
+    parser.add_argument("--get-top-benchmark", action="store_true",
+                        help="Print top benchmark location and stop.")
+    parser.add_argument("--get-benchmark", action="store_true",
+                        help="Print benchmark location for this particular install and stop.")
     parser.add_argument("--gpu", action="store_true", help="Run GPU tests.")
     parser.add_argument("--cpx", action="store_true", help="Enable CPX mode, allowing 6 GPUs per device.")
     options, unknown_options = parser.parse_known_args()
     if (options.atsHelp):
         subprocess.run(f"{ats_exe} --help", shell=True, check=True, text=True)
         return
-    if (options.get_benchmark):
+    if (options.get_top_benchmark):
         print(benchmark_dir)
         return
+
     #---------------------------------------------------------------------------
     # Setup machine info classes
     #---------------------------------------------------------------------------
@@ -144,6 +148,11 @@ def main():
         test_log_name = "gpu-"+test_log_name
     temp_uname = os.uname()
     hostname = temp_uname[1].rstrip("0123456789")
+    if (options.get_benchmark):
+        import SpheralConfigs
+        path = os.path.join(benchmark_dir, SpheralConfigs.configs(), hostname)
+        print(path)
+        return
     sys_type = os.getenv("SYS_TYPE")
     # Use ATS to for some machine specific functions
     if "MACHINE_TYPE" not in os.environ:

tests/run_perf.py

@@ -26,22 +26,15 @@
     num_threads = opts["threads"]
 
 # Adding --ciRun to the command line arguments of spheral-ats
-# triggers move of Caliper files to benchmark location
-benchmark_dir = None
 test_runs = 1 # Number of times to run each test
-CIRun = False
 if "cirun" in opts and opts["cirun"]:
-    CIRun = True
     test_runs = 3
-    benchmark_dir = opts["benchmark_dir"]
 is_rerun = False
 if "rerun" in opts and opts["rerun"]:
     is_rerun = True
 #---------------------------------------------------------------------------
 # Hardware configuration
 #---------------------------------------------------------------------------
-# This should be {$SYS_TYPE}_{compiler name}_{compiler version}_{mpi or cuda info}
-spheral_install_config = SpheralConfigs.config()
 mpi_enabled = SpheralConfigs.mpi_enabled()
 # Retrieve the host name and remove any numbers
 temp_uname = os.uname()
@@ -65,41 +58,20 @@
 def gather_files(manager):
     '''
     Function to gather Caliper file when ATS is finished running.
-    Used by ATS for gathering benchmark Caliper files.
     '''
-    instpath = os.path.join(benchmark_dir, spheral_install_config)
-    macpath = os.path.join(instpath, hostname)
-    outdir = os.path.join(macpath, "latest")
-    if (not is_rerun):
-        if (os.path.exists(outdir)):
-            # Move existing benchmark data to a different directory
-            log(f"Renaming existing {outdir} to {int(time.time())}", echo=True)
-            os.rename(outdir, os.path.join(macpath, f"{int(time.time())}"))
-        log(f"Creating {outdir}", echo=True)
-        os.makedirs(outdir)
     filtered = [test for test in manager.testlist if test.status is PASSED]
-    # Set read/write/execute permissions for owner and group
-    perms = stat.S_IRWXU | stat.S_IRWXG
+    logdir = log.directory
+    log(f"Copying caliper files to {logdir}")
     for test in filtered:
         run_dir = test.directory
         cali_filename = test.options["caliper_filename"]
         cfile = os.path.join(run_dir, cali_filename)
         test_name = test.options["label"]
-        outfile = os.path.join(outdir, cali_filename)
-        log(f"Moving {cali_filename} to {outdir}", echo=True)
-        if (CIRun):
-            shutil.move(cfile, outfile)
-            os.chmod(outfile, perms)
-            shutil.chown(outfile, group="sduser")
-    if (CIRun):
-        cpaths = [outdir, macpath, instpath, benchmark_dir]
-        for p in cpaths:
-            os.chmod(p, perms)
-            shutil.chown(p, group="sduser")
+        outfile = os.path.join(logdir, cali_filename)
+        shutil.move(cfile, outfile)
 
 # If running CI, run gather_files on exit
-if (benchmark_dir):
-    onExit(gather_files)
+onExit(gather_files)
 glue(keep=True, independent=True, ngpu=0, nt=1)
 
 #---------------------------------------------------------------------------