Ensure transitive dependencies for headers and libraries are always available

easybuild/tools/toolchain/toolchain.py

@@ -1191,6 +1191,25 @@ def _add_dependency_variables(self, names=None, cpp=None, ld=None):
             else:
                 dep_roots.extend(self.get_software_root(dep['name']))
 
+        # Exactly how the include headers or link libraries are going to be referenced is configuration dependent,
+        # gather all the options from the environment and add them as fallback options for the chosen method
+        # (allows for transitive dependencies)
+        paths_options = {'cpp_headers': [], 'linker': []}
+        for search_path_var, paths_list in paths_options.items():
+            for env_var in [y for x in SEARCH_PATH[search_path_var].values() for y in x if y.endswith('PATH')]:
+                paths_value = os.getenv(env_var)
+                if paths_value:
+                    self.log.debug(f"Determining fallback directories to retain from ${env_var}: {paths_value}")
+                    paths = paths_value.split(':')
+                    paths_list = unique_ordered_extend(paths_list, paths)
+                    # Now that we have the value, we make sure that EasyBuild will reset the environment variable later
+                    # and only use the configured option
+                    # (our RPATH wrappers rely on LIBRARY_PATH so we need an exception for that)
+                    if env_var != 'LIBRARY_PATH':
+                        self.variables.setdefault(env_var, append_empty=True)
+            for var in SEARCH_PATH[search_path_var][self.search_path[search_path_var]]:
+                self.variables.append_subdirs(var, '', subdirs=paths_list)
+
         for dep_root in dep_roots:
             self._add_dependency_cpp_headers(dep_root, extra_dirs=cpp)
             self._add_dependency_linker_paths(dep_root, extra_dirs=ld)
@@ -1221,6 +1240,14 @@ def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
             header_dirs = unique_ordered_extend(header_dirs, extra_dirs)
 
             self.log.info(f"Adding header paths to toolchain variable '{env_var}': {dep_root} (subdirs: {header_dirs})")
+            # it may already exist, in which case we remove it to add it back with higher priority
+            for header_dir in header_dirs:
+                if env_var in self.variables.keys():
+                    header_path = os.path.join(dep_root, header_dir)
+                    self.log.debug(f"Attempting to remove any previous instance of {header_path} from {env_var} list: "
+                                   f"{self.variables[env_var]}")
+                    self.variables[env_var].try_remove([header_path])
+                    self.log.debug(f"New value of {env_var} list: {self.variables[env_var]}")
             self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs)
 
     def _add_dependency_linker_paths(self, dep_root, extra_dirs=None):

test/framework/easyblock.py

@@ -3737,17 +3737,19 @@ def test_exts_deps_build_env(self):
         # put dummy modules in place where we can control $EBROOT value
         openmpi_fn = '4.1.5-GCC-12.3.0'
         zlib_fn = '1.2.13-GCCcore-12.3.0'
+        cuda_fn = '9.1.85'
         mod_files = [
             ('OpenMPI', openmpi_fn),
             ('zlib', zlib_fn),
+            ('CUDA', cuda_fn)
         ]
 
         test_mods = os.path.join(self.test_prefix, 'modules')
 
         for name, mod_fn in mod_files:
             mod_fp = os.path.join(testdir, 'modules', name, mod_fn)
 
-            header_fn = 'zlib.h' if name == 'zlib' else 'mpi.h'
+            header_fn = f'{name}.h' if name != 'OpenMPI' else 'mpi.h'
 
             dep_root = os.path.join(self.test_prefix, 'software', name, mod_fn)
             write_file(os.path.join(dep_root, 'include', header_fn), '')
@@ -3758,6 +3760,9 @@ def test_exts_deps_build_env(self):
             # add statement to inject extra subdirectory to $CPATH,
             # which is supposed to be retained in build environment
             mod_txt += f'\nprepend-path\tCPATH\t$root/include/{name}'
+            # Add a transitive dependency to zlib
+            if name == 'zlib':
+                mod_txt += f'\nmodule load CUDA/{cuda_fn}'
 
             test_mod_file = os.path.join(test_mods, name, mod_fn)
             write_file(test_mod_file, mod_txt)
@@ -3795,6 +3800,8 @@ def test_exts_deps_build_env(self):
                     f'software/OpenMPI/{openmpi_fn}/include',
                     f'software/zlib/{zlib_fn}/include/zlib',
                     f'software/zlib/{zlib_fn}/include',
+                    f'software/CUDA/{cuda_fn}/include/CUDA',
+                    f'software/CUDA/{cuda_fn}/include',
                 ]
                 if env_var.endswith('PATH'):
                     regex = re.compile(f'^{env_var}=' + ':'.join('[^ ]+/' + p for p in paths) + '$', re.M)