Merge pull request #498 from llnl/395-gpu-performance-tests

Simplify performance benchmark data

Author: ldowen

.gitlab/performance.yml

@@ -15,15 +15,16 @@
     - cd $INSTALL_DIR
     - INSTALL_BENCHMARK_DIR=$(./bin/spheral-ats --get-benchmark)
     - PERF_OUT_DIR=$INSTALL_BENCHMARK_DIR/$(date +%s)
-    - mkdir $PERF_OUT_DIR
+    - echo "Creating benchmark directory at ${PERF_OUT_DIR}"
+    - mkdir -p $PERF_OUT_DIR
     - echo "Running performance tests using install at $INSTALL_DIR"
     - rm -rf perf_logs gpu_perf_logs
     - ./bin/spheral-ats --ciRun --numNodes 2 --logs perf_logs $INSTALL_DIR/$PERF_ATS_FILE
     - cp perf_logs/*.cali $PERF_OUT_DIR
     - ./bin/spheral-ats --gpu --ciRun --numNodes 2 --logs gpu_perf_logs $INSTALL_DIR/$PERF_ATS_FILE
     # Copy caliper files to the benchmark directory and don't complain if you cant
     - cp gpu_perf_logs/*.cali $PERF_OUT_DIR 2>/dev/null || true
-    - chmod -R gu+rwx $PERF_OUT_DIR
+    - chmod -R gu+rx $PERF_OUT_DIR
     - chgrp -R sduser $PERF_OUT_DIR
 
 toss_gcc_mpi_perf:

.gitlab/scripts.yml

@@ -65,24 +65,25 @@
 
 # This copies the install so scheduled perf tests can use it
 # Adding test-perf to commit message will trigger the perf install from that branch
+# Requires upstream TPLs to be up to date
 .perf_install:
   extends: [.retry_rule]
   stage: perf_install
   variables:
     GIT_STRATEGY: none
   rules:
-    - if: '($CI_COMMIT_MESSAGE =~ /test-perf/ || $CI_COMMIT_REF_NAME == "develop")'
+    - !reference [.merge_pr_rule, rules]
+    - if: '$CI_COMMIT_MESSAGE =~ /test-perf/'
       when: always
   script:
+    - umask 027
     - CI_BUILD_DIR=$(cat ci-dir.txt)
     - INSTALL_DIR=$PERF_INSTALL_DIR/${SPEC//%/_}
     - ml load mpifileutils
     - echo "Doing install in ${INSTALL_DIR}"
     - if [[ -d $INSTALL_DIR ]]; then $RUN_CMD -n $NPROC drm $INSTALL_DIR; else echo "No install exists."; fi
-    - cd $CI_BUILD_DIR/build_gitlab/build
-    - cmake ../../ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR
-    - make -j $NPROC install
-    - $RUN_CMD -n $NPROC dchmod --mode go+rx $INSTALL_DIR
+    - cd $CI_BUILD_DIR
+    - ./$SCRIPT_DIR/devtools/host-config-build.py --no-clean --build --nprocs $NPROC --host-config gitlab.cmake --install-dir $INSTALL_DIR
   artifacts:
     paths:
       - ci-dir.txt
@@ -136,7 +137,7 @@
   stage: update_tpls
   rules:
     - !reference [.merge_pr_rule, rules]
-    - if: '$CI_COMMIT_MESSAGE =~ /update-tpls/'
+    - if: '($CI_COMMIT_MESSAGE =~ /update-tpls/ || $CI_COMMIT_MESSAGE =~ /test-perf/)'
   script:
     - umask 027
     - ./$SCRIPT_DIR/devtools/tpl-manager.py --update-upstream

CMakeLists.txt

@@ -1,6 +1,6 @@
 # CMakeLists to build the Spheral library.
 
-cmake_minimum_required(VERSION 3.21)
+cmake_minimum_required(VERSION 3.24)
 include(cmake/SpheralVersion.cmake)
 project(spheral LANGUAGES C CXX Fortran VERSION ${SPHERAL_VERSION})
 

RELEASE_NOTES.md

@@ -24,6 +24,7 @@ Notable changes include:
     * Added a page to the docs about GPU development. 
     * Optimized field lookups in state, reducing per-call cost from O(N) to O(log N)
     * Added the more aptly named SPHERAL_EXTERNAL_INSTALL in places where ENABLE_STATIC_TPLS was being used.
+    * Require minimum CMake version 3.24.
 
   * Bug fixes:
     * Adiak memory leak is fixed by calling adiak::clean() before exit.
@@ -52,6 +53,7 @@ Notable changes include:
       * Added umask command for updating upstreams and removed separate job to update permissions.
       * Build and test job now fails if import of Spheral module fails.
       * Number of ranks on RZAdams increased from 84 to 96 per node for performance tests.
+      * Created advanced timers for lower level timers. These are disabled by default for cleaner Caliper timer trees.
     * Builds and installs are cleaner:
       * Rpaths are no longer overwritten, allowing things set in the Spack host config file to be used.
       * Spheral libraries are only installed once now and a Spheral.pth with a relative path to the install lib is used in the virtual python environment.

docs/developer/dev/diagnostic_tools.rst

@@ -78,18 +78,32 @@ The following macros are used to create timing regions in the Spheral C++ interf
 
 - ``TIME_BEGIN("timer_name")`` and ``TIME_END("timer_name")`` create a region between the two different calls and use the string (in this case ``timer_name``) as the name.
 
+It is not recommended to add timers to lower level functions. For example, functions that are called on a per SPH node basis should not have timers. If timers are desired for these types of functions, use advanced timers, denoted as ``ADV_TIME_BEGIN("timer_name")`` and ``ADV_TIME_END("timer_name")``. Advanced timers must be enabled through the python interface, as described below.
 
 Adding Region Timers in Python
 ------------------------------
 
 Region timers can be added inside the python code using the following function calls:
 ::
 
-   from SpheralUtilities import TimerMgr
+   from SpheralCompiledModules.SpheralUtilities import TimerMgr
+   if (not TimerMgr.is_started()):
+        TimerMgr.default_start("TestName.cali")
    TimerMgr.timer_begin("timer_name")
    some_function_call()
    TimerMgr.timer_end("timer_name")
 
+Similarly, advanced timers can be added through the python interface. They must be enabled through the python interface for any advanced timers to be included:
+::
+
+   from SpheralCompiledModules.SpheralUtilities import TimerMgr
+   if (not TimerMgr.is_started()):
+        TimerMgr.default_start("TestName.cali")
+   TimerMgr.enable_advanced_timers() # Required for advanced timers to be included
+   TimerMgr.adv_timer_begin("timer_name")
+   some_function_call()
+   TimerMgr.adv_timer_end("timer_name")
+
 .. note::
    All timers must have both a start and end call. Otherwise, memory issues will occur.
 
@@ -129,7 +143,7 @@ Starting Caliper Manually
 As mentioned above, the Caliper timing manager is normally configured and started in the ``commandLine()`` routine. However, Caliper can be directly configured and started through the python interface, if desired. This can be done by putting the following into the python file:
 ::
 
-   from SpheralUtilities import TimerMgr
+   from SpheralCompiledModules.SpheralUtilities import TimerMgr
    caliper_config = "some_configuration(output=some_filename.txt)"
    TimerMgr.add(caliper_config)
    TimerMgr.start()
@@ -184,7 +198,7 @@ The general procedure to comparing performance regression tests is:
    If ``run_perf.py`` is run on a non-MPI Spheral build, it will only use 1 rank and will thread all other available cores.
 
 
-To simply display the Thicket timing tree for a performance run without doing any sort of comparison, use:
+To simply display the Thicket timing tree for a performance run without doing any comparison, use:
 ::
 
    ./bin/spheral ./scripts/performance_analysis.py --perfdata /path/to/perf/data

scripts/devtools/host-config-build.py

@@ -57,22 +57,28 @@ def main():
     build_dir="{0}/build_{1}".format(source_dir, hostconfig)
   else:
     build_dir="{0}/{1}".format(source_dir, args.build_dir)
-  if not args.install_dir:
-    install_dir="{0}/install".format(build_dir)
+
+  # Set install directory, give preference to the CMAKE_INSTALL_PREFIX
+  if args.D and any("CMAKE_INSTALL_PREFIX" in x for x in args.D):
+    install_dir = None
+  elif args.install_dir:
+    install_dir = args.install_dir
   else:
-    install_dir=args.install_dir
+    install_dir="{0}/install".format(build_dir)
+
   build_dir=build_dir+"/build"
   # Pull the cmake command to use out of our host config.
   cmake_cmd=sexe("grep 'CMake executable' \"{0}\"".format(hostconfig_path), ret_output=True, echo=False).split()[-1]
 
   cmake_extra_args=""
   if args.D and args.D != ['']:
     cmake_extra_args="-D"+" -D".join(args.D)
-  
+
   print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
   print("~ Host-config:      {0}".format(hostconfig_path))
   print("~ Build Dir:        {0}".format(build_dir))
-  print("~ Install Dir:      {0}".format(install_dir))
+  if install_dir:
+    print("~ Install Dir:      {0}".format(install_dir))
   print("~ Source Dir:       {0}".format(source_dir))
   print("~ Cmake cmd:        {0}".format(cmake_cmd))
   print("~ Extra CMake Args: {0}".format(cmake_extra_args))
@@ -82,15 +88,18 @@ def main():
   # Clean our build and install dirs...
   if not args.no_clean:
       sexe("rm -rf \"{0}\" 2>/dev/null".format(build_dir),echo=True)
-      sexe("rm -rf \"{0}\" 2>/dev/null".format(install_dir),echo=True)
+      if install_dir:
+        sexe("rm -rf \"{0}\" 2>/dev/null".format(install_dir),echo=True)
   sexe("mkdir -p \"{0}\"".format(build_dir),echo=True)
-  sexe("mkdir -p \"{0}\"".format(install_dir),echo=True)
+  if install_dir:
+    sexe("mkdir -p \"{0}\"".format(install_dir),echo=True)
 
   # Move to the build directory.
   os.chdir(build_dir)
 
   # Run our CMake config step.
-  sexe(f"{cmake_cmd} -C {hostconfig_path} -DCMAKE_INSTALL_PREFIX={install_dir} {cmake_extra_args} {source_dir}")
+  install_str = f"-DCMAKE_INSTALL_PREFIX={install_dir}" if install_dir else ""
+  sexe(f"{cmake_cmd} -C {hostconfig_path} {install_str} {cmake_extra_args} {source_dir}")
 
   # Build and install Spheral
 

scripts/devtools/performance_analysis.py

@@ -136,7 +136,7 @@ def compare_config(cdata, rdata):
     Do not use if comparing across hardware or install configurations.
     Assumes Thicket data has comparable test_name, total_internal_nodes, and total_steps
     """
-    hardware_tests = ["install_config", "cluster", "jobsize", "threads_per_rank"]
+    hardware_tests = ["spec", "cluster", "jobsize", "threads_per_rank"]
     return compare_metadata(cdata, rdata, hardware_tests)
 
 def filter_tests(data, test_name):
@@ -271,9 +271,9 @@ def main():
                     raise Exception("No reference or benchmark data specified")
             # If using benchmark reference data, only grab the current install/machine
             # Get install config and machine name from current data
-            install_config = curdata.metadata["install_config"].iloc[0]
+            install_spec = curdata.metadata["spec"].iloc[0]
             machine_name = curdata.metadata["cluster"].iloc[0]
-            ref_loc = os.path.join(ref_files, install_config, machine_name)
+            ref_loc = os.path.join(ref_files, machine_name, install_spec)
             if (not os.path.exists(ref_loc)):
                 raise Exception(f"Benchmark location {ref_loc} does not exists")
             cali_ref_files = glob.glob(os.path.join(ref_loc, "**/*.cali"), recursive=True)

scripts/lc/hist_data_utils.py

@@ -12,7 +12,7 @@ def create_header(title, file_dict):
 <body>
   <form>
     <select onchange="window.location.href = this.value;">
-      <option value="">-- Select machine/config --</option>
+      <option value="">-- Select machine --</option>
 """
     for link, ff in file_dict.items():
         html_string += " "*6

scripts/lc/plot_hist_data.py

@@ -63,6 +63,15 @@ def extract_data(cali_file):
                         rtimes[rg] = tval
     return runday, rtimes
 
+def update_spec(spec):
+    "Spec names changes from spack 0.12 to 1.0, so we must update the names to match"
+    repl_dict = [("clang", "llvm"), ("rocmcc", "llvm-amdgpu"),
+                 ("cray-mpich", "mpi"), ("mvapich2", "mpi")]
+    newspec = spec
+    for i in repl_dict:
+        newspec = newspec.replace(i[0], i[1])
+    return newspec
+
 def get_globals(cali_file):
     # If cali_file is just a dir, grab the latest caliper file
     if type(cali_file) == list:
@@ -83,7 +92,9 @@ def get_spec(cali_file):
     return gls["spec"]
 
 def compare_time(cfile, num_of_months):
-    tdelta = (time.time() - os.path.getmtime(cfile))/ (60*60*24)
+    # The benchmark file times might not be accurate, use file names instead
+    file_mtime = int(cfile.split("_")[-1].split(".")[0])
+    tdelta = (time.time() - file_mtime)/ (60*60*24)
     if (tdelta < 30*num_of_months):
         return True
     return False
@@ -101,19 +112,12 @@ def split_test_name(test_name, test_base_names):
             return base, variant
     return None, None
 
-def update_spec(spec):
-    "Spec names changes from spack 0.12 to 1.0, so we must update the names to match"
-    newspec = spec.replace("clang", "llvm")
-    newspec = newspec.replace("rocmcc", "llvm-amdgpu")
-    return newspec
-
 # Convert array of dictionaries into panda dataframe
 def convert_to_dataframe(in_array, test_base_names):
     data = []
     for in_data in in_array:
         mac_name = in_data["machine"]
         spec_name = update_spec(in_data["spec"])
-        config = in_data["config"]
         data_dict = in_data["data_dict"]
         for test_name, tdata in data_dict.items():
             test_base, test_var = split_test_name(test_name, test_base_names)
@@ -123,7 +127,6 @@ def convert_to_dataframe(in_array, test_base_names):
                 for reg, val in time_dict.items():
                     data.append({"Machine": mac_name,
                                  "Spec": spec_name,
-                                 "Config": config,
                                  "Test Name": test_name,
                                  "Test Base": test_base,
                                  "Test Var": test_var,
@@ -210,29 +213,28 @@ def main():
     if (args.test):
         test_names = [test_names[0]]
     test_bases = pt.get_test_bases()
-    # Benchmark directory is organized as /configs/machines/run_dates/*.cali
+    # Benchmark directory is organized as /machine/spec/run_dates/*.cali
     # Each set of caliper files contain a single spec
-    # Multiple configs could span a single machine+spec
-    config_dirs = glob.glob(os.path.join(args.bench, "*"))
+    mac_dirs = glob.glob(os.path.join(args.bench, "*"))
     # Extract all machine directories
     all_mac_specs = {}
     all_specs = {}
     if (rank == 0):
-        for cc in config_dirs:
-            mac_dirs = glob.glob(os.path.join(cc, "*"))
-            for mc_path in mac_dirs:
-                mac_name = os.path.basename(mc_path)
-                cspec = get_spec(mc_path)
+        for mc_path in mac_dirs:
+            mac_name = os.path.basename(mc_path)
+            spec_dirs = glob.glob(os.path.join(mc_path, "*"))
+            for cc in spec_dirs:
+                cspec = os.path.basename(cc)
                 key = mac_name+" "+cspec
                 all_specs[key] = cspec
-                cur_path = os.path.join(mc_path, "*")
+                cur_path = os.path.join(cc, "*")
                 if key in all_mac_specs:
                     all_mac_specs[key].append(cur_path)
                 else:
                     all_mac_specs.update({key: [cur_path]})
         if (args.local_run_dir):
             local_cali_files = glob.glob(os.path.join(args.local_run_dir, "*.cali"))
-            cspec = get_spec(local_cali_files)
+            cspec = update_spec(get_spec(local_cali_files))
             mac_name = get_host(local_cali_files)
             key = mac_name+" "+cspec
             if key in all_mac_specs:

scripts/spack/configs/toss_4_x86_64_ib/packages.yaml

@@ -62,9 +62,9 @@ packages:
 
   mvapich2:
     externals:
-    - spec: mvapich2@2.3.6.gcc process_managers=slurm %gcc@10.3.1
+    - spec: mvapich2@2.3.6 process_managers=slurm %gcc@10.3.1
       prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.6-gcc-10.3.1-magic
-    - spec: mvapich2@2.3.7.llvm process_managers=slurm %clang@19.1.3
+    - spec: mvapich2@2.3.7 process_managers=slurm %clang@19.1.3
       prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-19.1.3
     buildable: false