Make matching component sources in Bundle easyblock more reliable

Store where sources have been added to get the corresponding filled
structure without relying on partially resolved templates.

Author: Flamefire

easybuild/easyblocks/generic/bundle.py

@@ -105,6 +105,10 @@ def __init__(self, *args, **kwargs):
         # (like adding component sources to top-level sources easyconfig parameter)
         self.cfg = self.cfg.copy()
 
+        # Keep track of sources for each component to restore them:
+        # (Component instance, start idx [in self.src], end idx)
+        self.comp_to_src_idxs = []
+
         # disable templating to avoid premature resolving of template values
         # Note that self.cfg.update also resolves templates!
         with self.cfg.disable_templating():
@@ -197,6 +201,10 @@ def __init__(self, *args, **kwargs):
 
                 comp_cfg.generate_template_values()
 
+                # Combine all component sources into the top-level sources parameter
+                # This allows reusing top-level source_urls and unpacking them all in the extract_step
+
+                old_num_srcs = len(self.cfg.get('sources', resolve=False))
                 # Don't require that all template values can be resolved at this point but still resolve them.
                 # This is important to ensure that template values like %(name)s and %(version)s
                 # are correctly resolved with the component name/version before values are copied over to self.cfg
@@ -226,21 +234,9 @@ def __init__(self, *args, **kwargs):
                 comp_checksums = comp_cfg['checksums']
                 if comp_checksums:
                     src_cnt = len(comp_sources)
-
                     # add per-component checksums for sources to list of checksums
                     self.cfg.update('checksums', comp_checksums[:src_cnt])
 
-                    # add per-component checksums for patches to list of checksums for patches
-                    checksums_patches.extend(comp_checksums[src_cnt:])
-
-                with comp_cfg.allow_unresolved_templates():
-                    comp_patches = comp_cfg['patches']
-                    comp_postinstall_patches = comp_cfg['postinstallpatches']
-                if comp_patches:
-                    self.cfg.update('patches', comp_patches)
-                    # Patch step is skipped so adding postinstall patches of components here is harmless
-                    self.cfg.update('patches', comp_postinstall_patches)
-
                 # instantiate the component to transfer further information
                 comp_instance = comp_cfg.easyblock(comp_cfg, logfile=self.logfile)
 
@@ -253,6 +249,11 @@ def __init__(self, *args, **kwargs):
                     self.comp_cfgs_sanity_check.append(comp_instance)
                 # lastly, add it to the list of components we'll deal with later
                 self.comp_instances.append((comp_cfg, comp_instance))
+                new_num_srcs = len(self.cfg.get('sources', resolve=False))
+                self.comp_to_src_idxs.append((self.comp_instances[-1], old_num_srcs, new_num_srcs))
+                # check if sanity checks are enabled for the component
+                if self.cfg['sanity_check_all_components'] or comp_cfg['name'] in self.cfg['sanity_check_components']:
+                    self.comp_cfgs_sanity_check.append(self.comp_instances[-1])
 
             self.cfg.update('checksums', checksums_patches + orig_checksums)
 
@@ -274,14 +275,29 @@ def check_checksums(self):
 
         return checksum_issues
 
+    def fetch_step(self):
+        """Fetch sources of all extensions"""
+        super().fetch_step()
+        # Init src attribute as usually done by fetch_step
+        for (_, comp), start_idx, end_idx in self.comp_to_src_idxs:
+            comp.src = self.src[start_idx:end_idx]
+            # need to run fetch_patches to ensure per-component patches are gathered
+            comp.fetch_patches()
+
     def prepare_step(self, *args, **kwargs):
         """
         Pre-configure step.
-        At this point, dependencies are known. So transfer them to all components.
+        At this point, dependencies & properties are known. So transfer them to all components.
         """
-        super().prepare_step(self, *args, **kwargs)
+        super().prepare_step(*args, **kwargs)
         for _, comp in self.comp_instances:
             comp.toolchain.dependencies = self.toolchain.dependencies
+            # correct build/install dirs
+            comp.builddir = self.builddir
+            comp.install_subdir, comp.installdir = self.install_subdir, self.installdir
+
+            # make sure we can build in parallel
+            comp.set_parallel()
 
     def patch_step(self):
         """Patch step must be a no-op for bundle, since there are no top-level sources/patches."""
@@ -339,43 +355,18 @@ def install_step(self):
                       (comp.name, comp.version, idx + 1, comp_cnt))
             self.log.info("Installing component %s v%s using easyblock %s", comp.name, comp.version, cfg.easyblock)
 
-            # make sure we can build in parallel
-            comp.set_parallel()
-
+(??)            # make sure we can build in parallel
+(??)            comp.set_parallel()
+(??)
             # figure out correct start directory
-            comp.guess_start_dir()
-
-            # need to run fetch_patches to ensure per-component patches are applied
-            comp.fetch_patches()
-
+            # Compatibility with ECs expecting the previous behavior where src wasn't populated at this point
+            tmp_src = comp.src
             comp.src = []
+            comp.guess_start_dir()
+            comp.src = tmp_src
 
-            # find matching entries in self.src for this component
-            with comp.cfg.allow_unresolved_templates():
-                comp_sources = comp.cfg['sources']
-            for source in comp_sources:
-                if isinstance(source, str):
-                    comp_src_fn = source
-                elif isinstance(source, dict):
-                    if 'filename' in source:
-                        comp_src_fn = source['filename']
-                    else:
-                        raise EasyBuildError("Encountered source file specified as dict without 'filename': %s", source)
-                else:
-                    raise EasyBuildError("Specification of unknown type for source file: %s", source)
-
-                found = False
-                for src in self.src:
-                    if src['name'] == comp_src_fn:
-                        self.log.info("Found spec for source %s for component %s: %s", comp_src_fn, comp.name, src)
-                        comp.src.append(src)
-                        found = True
-                        break
-                if not found:
-                    raise EasyBuildError("Failed to find spec for source %s for component %s", comp_src_fn, comp.name)
-
-                # location of first unpacked source is used to determine where to apply patch(es)
-                comp.src[-1]['finalpath'] = comp.cfg['start_dir']
+            # location of first unpacked source is used to determine where to apply patch(es)
+            comp.src[0]['finalpath'] = comp.cfg['start_dir']
 
             self._install_component(comp)