Merge branch 'main' into zen5

Author: laraPPr

.github/workflows/tests_eessi_extend_module.yml

@@ -50,9 +50,13 @@ jobs:
           export MY_INSTALLATION_PATH=/tmp/easybuild
           export EASYBUILD_PREFIX=$MY_INSTALLATION_PATH
           eb EESSI-extend-easybuild.eb --rebuild
+          module unload EasyBuild
 
           # Verify that we can pick the installed version up
           module use $MY_INSTALLATION_PATH/modules/all
+          # Verify we can run "module show" on the module
+          module show EESSI-extend/${{matrix.eessi_version}}-easybuild
+          # Now load it and run additional checks
           module load EESSI-extend/${{matrix.eessi_version}}-easybuild
           echo $EBROOTEESSIMINEXTEND | grep $MY_INSTALLATION_PATH || { echo "ERROR: Installed version of EESSI-extend not picked up (loaded $EBROOTEESSIMINEXTEND)" >&2; exit 1; }
           # Do a test rebuild (with default and local hooks)

EESSI-extend-easybuild.eb

@@ -133,7 +133,7 @@ if eessi_cvmfs_install then
     LmodError("You cannot use EESSI_CVMFS_INSTALL in combination with any other EESSI_*_INSTALL environment variables")
   end
   easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH")
-  -- enforce accelerator subdirectory usage for CVMFS installs (only if an accelerator install is requested)
+  -- Enforce accelerator subdirectory usage for CVMFS installs (only if an accelerator install is requested)
   if (eessi_accelerator_target ~= nil) and (cuda_compute_capability ~= nil or amd_compute_capability ~= nil) and (os.getenv("EESSI_ACCELERATOR_INSTALL") ~= nil) then
       easybuild_installpath = pathJoin(easybuild_installpath, eessi_accelerator_target)
   end
@@ -143,7 +143,7 @@ elseif eessi_site_install then
     LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables")
   end
   easybuild_installpath = os.getenv("EESSI_SITE_SOFTWARE_PATH")
-  -- enforce accelerator subdirectory usage for site installs (only if an accelerator install is requested)
+  -- Enforce accelerator subdirectory usage for site installs (only if an accelerator install is requested)
   if (eessi_accelerator_target ~= nil) and (cuda_compute_capability ~= nil) and (os.getenv("EESSI_ACCELERATOR_INSTALL") ~= nil) then
       easybuild_installpath = pathJoin(easybuild_installpath, eessi_accelerator_target)
   end
@@ -232,7 +232,8 @@ end
 
 -- Make sure EasyBuild itself is loaded
 -- need to also handle the unload behaviour where the version is defined only before we unload
-easybuild_version = os.getenv("EBVERSIONEASYBUILD")
+-- (also provide a sensible default for cases like 'module show' where the modules are never loaded at all)
+easybuild_version = os.getenv("EBVERSIONEASYBUILD") or "5.1.1"
 if not ( isloaded("EasyBuild") ) then
     load(latest("EasyBuild"))
 end

bot/build.sh

@@ -193,8 +193,9 @@ else
 fi
 echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${EESSI_ACCELERATOR_TARGET_OVERRIDE}'"
 
-# Log the full lscpu and os-release info:
+# Log the full lscpu, ulimits, and os-release info:
 lscpu > _bot_job${SLURM_JOB_ID}.lscpu
+ulimit -a > _bot_job${SLURM_JOB_ID}.ulimits
 cat /etc/os-release > _bot_job${SLURM_JOB_ID}.os
 
 # Also: fetch CPU flags into an array, so that we can implement a hard check against a reference

eb_hooks.py

@@ -16,12 +16,12 @@
 from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
 from easybuild.tools.config import build_option, install_path, update_build_option
 from easybuild.tools.filetools import apply_regex_substitutions, copy_dir, copy_file, remove_file, symlink, which
+from easybuild.tools.modules import get_software_root, get_software_root_env_var_name
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import AARCH64, POWER, X86_64, det_parallelism, get_cpu_architecture, get_cpu_features
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
 from easybuild.tools.toolchain.toolchain import is_system_toolchain
 from easybuild.tools.version import VERSION as EASYBUILD_VERSION
-from easybuild.tools.modules import get_software_root_env_var_name
 
 # prefer importing LooseVersion from easybuild.tools, but fall back to distuils in case EasyBuild <= 4.7.0 is used
 try:
@@ -52,6 +52,7 @@
 
 EESSI_INSTALLATION_REGEX = r"^/cvmfs/[^/]*.eessi.io/versions/"
 HOST_INJECTIONS_LOCATION = "/cvmfs/software.eessi.io/host_injections/"
+SITE_INSTALLATION_LOCATION = os.getenv("EESSI_SITE_SOFTWARE_PREFIX", HOST_INJECTIONS_LOCATION)
 
 # Make sure a single environment variable name is used for this throughout the hooks
 EESSI_IGNORE_ZEN4_GCC1220_ENVVAR="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220"
@@ -744,7 +745,7 @@ def pre_fetch_hook_check_installation_path(self, *args, **kwargs):
     accelerator_toolchains = ['rocm-compilers', 'rompi', 'rfbf', 'rfoss']
     strict_eessi_installation = (
         bool(re.search(EESSI_INSTALLATION_REGEX, self.installdir)) or
-        self.installdir.startswith(HOST_INJECTIONS_LOCATION))
+        self.installdir.startswith(SITE_INSTALLATION_LOCATION))
     if strict_eessi_installation and not os.getenv("EESSI_OVERRIDE_STRICT_INSTALLPATH_CHECK"):
         dependency_names = self.cfg.dependency_names()
         if (
@@ -757,15 +758,19 @@ def pre_fetch_hook_check_installation_path(self, *args, **kwargs):
                 raise EasyBuildError(
                     f"It seems you are trying to install an accelerator package {self.cfg.name} into a "
                     f"non-accelerator location {self.installdir}. You need to reconfigure your installation to target "
-                    "the correct location."
+                    "the correct location. If using the EESSI-extend module, this means reloading that module "
+                    "with EESSI_ACCELERATOR_INSTALL set:\n"
+                    "  EESSI_ACCELERATOR_TARGET=1 module load EESSI-extend"
                     )
         else:
             # If we don't have an accelerator dependency then we should be in a CPU installation path
             if "/accel/" in self.installdir:
                 raise EasyBuildError(
                     f"It seems you are trying to install a CPU-only package {self.cfg.name} into accelerator location "
                     f"{self.installdir}. If this is a dependency of the package you are really interested in you will "
-                    "need to first install the CPU-only dependencies of that package."
+                    "need to first install the CPU-only dependencies of that package. If using the EESSI-extend "
+                    "module, this means reloading that module with EESSI_ACCELERATOR_INSTALL unset:\n"
+                    "  unset EESSI_ACCELERATOR_TARGET=1 && module load EESSI-extend"
                     )
 
 
@@ -1480,6 +1485,23 @@ def pre_configure_hook_cmake_system(self, *args, **kwargs):
         raise EasyBuildError("CMake-specific hook triggered for non-CMake easyconfig?!")
 
 
+def pre_configure_hook_Zoltan(self, *args, **kwargs):
+    """
+    Pre-configure hook for Zoltan to filter out ParMETIS configure options,
+    since we filter out ParMETIS as a dependency
+    """
+    if self.name == 'Zoltan':
+        if get_software_root('ParMETIS') is None:
+            configopts = self.cfg['configopts']
+            # get rid of all --with-parmetis configure options, and inject --without-parmetis
+            configopts = re.sub('--with-parmetis[^ ]*', '', configopts)
+            configopts += " --without-parmetis"
+            self.cfg['configopts'] = configopts
+            self.log.info("Removed --with-parmetis* configure options for {self.name}, ParMETIS is not a dependency")
+    else:
+        raise EasyBuildError("Zoltan-specific hook triggered for non-Zoltan easyconfig?!")
+
+
 def pre_test_hook(self, *args, **kwargs):
     """Main pre-test hook: trigger custom functions based on software name."""
     if self.name in PRE_TEST_HOOKS:
@@ -2068,24 +2090,25 @@ def post_easyblock_hook(self, *args, **kwargs):
 
 PRE_CONFIGURE_HOOKS = {
     'BLIS': pre_configure_hook_BLIS,
+    'CMake': pre_configure_hook_cmake_system,
     'CUDA-Samples': pre_configure_hook_CUDA_Samples_test_remove,
-    'GObject-Introspection': pre_configure_hook_gobject_introspection,
+    'Dyninst': pre_configure_hook_dyninst,
     'Extrae': pre_configure_hook_extrae,
+    'GObject-Introspection': pre_configure_hook_gobject_introspection,
     'Graphviz': pre_configure_hook_graphviz,
     'GRASS': pre_configure_hook_grass,
+    'LAMMPS': pre_configure_hook_LAMMPS_zen4_and_aarch64_cuda,
     'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic,
     'LLVM': pre_configure_hook_llvm,
-    'ROCm-LLVM': pre_configure_hook_llvm,
     'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep,
     'OpenBLAS': pre_configure_hook_openblas_optarch_generic,
     'OpenMPI': pre_configure_hook_openmpi_ipv6,
     'PMIx': pre_configure_hook_pmix_ipv6,
     'PRRTE': pre_configure_hook_prrte_ipv6,
-    'WRF': pre_configure_hook_wrf_aarch64,
-    'LAMMPS': pre_configure_hook_LAMMPS_zen4_and_aarch64_cuda,
+    'ROCm-LLVM': pre_configure_hook_llvm,
     'Score-P': pre_configure_hook_score_p,
-    'Dyninst': pre_configure_hook_dyninst,
-    'CMake': pre_configure_hook_cmake_system,
+    'WRF': pre_configure_hook_wrf_aarch64,
+    'Zoltan': pre_configure_hook_Zoltan,
 }
 
 PRE_TEST_HOOKS = {