diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index a8c7c2a259..15ef47c6ea 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -1191,6 +1191,25 @@ def _add_dependency_variables(self, names=None, cpp=None, ld=None): else: dep_roots.extend(self.get_software_root(dep['name'])) + # Exactly how the include headers or link libraries are going to be referenced is configuration dependent, + # gather all the options from the environment and add them as fallback options for the chosen method + # (allows for transitive dependencies) + paths_options = {'cpp_headers': [], 'linker': []} + for search_path_var, paths_list in paths_options.items(): + for env_var in [y for x in SEARCH_PATH[search_path_var].values() for y in x if y.endswith('PATH')]: + paths_value = os.getenv(env_var) + if paths_value: + self.log.debug(f"Determining fallback directories to retain from ${env_var}: {paths_value}") + paths = paths_value.split(':') + paths_list = unique_ordered_extend(paths_list, paths) + # Now that we have the value, we make sure that EasyBuild will reset the environment variable later + # and only use the configured option + # (our RPATH wrappers rely on LIBRARY_PATH so we need an exception for that) + if env_var != 'LIBRARY_PATH': + self.variables.setdefault(env_var, append_empty=True) + for var in SEARCH_PATH[search_path_var][self.search_path[search_path_var]]: + self.variables.append_subdirs(var, '', subdirs=paths_list) + for dep_root in dep_roots: self._add_dependency_cpp_headers(dep_root, extra_dirs=cpp) self._add_dependency_linker_paths(dep_root, extra_dirs=ld) @@ -1221,6 +1240,14 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None): header_dirs = unique_ordered_extend(header_dirs, extra_dirs) self.log.info(f"Adding header paths to toolchain variable '{env_var}': {dep_root} (subdirs: {header_dirs})") + # it may already exist, in which case we remove it to add it back with higher priority + for header_dir in header_dirs: + if env_var in self.variables.keys(): + header_path = os.path.join(dep_root, header_dir) + self.log.debug(f"Attempting to remove any previous instance of {header_path} from {env_var} list: " + f"{self.variables[env_var]}") + self.variables[env_var].try_remove([header_path]) + self.log.debug(f"New value of {env_var} list: {self.variables[env_var]}") self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs) def _add_dependency_linker_paths(self, dep_root, extra_dirs=None): diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 408d4b1af5..7ce050f8d0 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -3737,9 +3737,11 @@ def test_exts_deps_build_env(self): # put dummy modules in place where we can control $EBROOT value openmpi_fn = '4.1.5-GCC-12.3.0' zlib_fn = '1.2.13-GCCcore-12.3.0' + cuda_fn = '9.1.85' mod_files = [ ('OpenMPI', openmpi_fn), ('zlib', zlib_fn), + ('CUDA', cuda_fn) ] test_mods = os.path.join(self.test_prefix, 'modules') @@ -3747,7 +3749,7 @@ def test_exts_deps_build_env(self): for name, mod_fn in mod_files: mod_fp = os.path.join(testdir, 'modules', name, mod_fn) - header_fn = 'zlib.h' if name == 'zlib' else 'mpi.h' + header_fn = f'{name}.h' if name != 'OpenMPI' else 'mpi.h' dep_root = os.path.join(self.test_prefix, 'software', name, mod_fn) write_file(os.path.join(dep_root, 'include', header_fn), '') @@ -3758,6 +3760,9 @@ def test_exts_deps_build_env(self): # add statement to inject extra subdirectory to $CPATH, # which is supposed to be retained in build environment mod_txt += f'\nprepend-path\tCPATH\t$root/include/{name}' + # Add a transitive dependency to zlib + if name == 'zlib': + mod_txt += f'\nmodule load CUDA/{cuda_fn}' test_mod_file = os.path.join(test_mods, name, mod_fn) write_file(test_mod_file, mod_txt) @@ -3795,6 +3800,8 @@ def test_exts_deps_build_env(self): f'software/OpenMPI/{openmpi_fn}/include', f'software/zlib/{zlib_fn}/include/zlib', f'software/zlib/{zlib_fn}/include', + f'software/CUDA/{cuda_fn}/include/CUDA', + f'software/CUDA/{cuda_fn}/include', ] if env_var.endswith('PATH'): regex = re.compile(f'^{env_var}=' + ':'.join('[^ ]+/' + p for p in paths) + '$', re.M)