Merge pull request #1321 from douglasjacobsen/package-injection

Allows software_spec's to be injected if not present in environments

Author: douglasjacobsen

lib/ramble/ramble/experiment_set.py

@@ -328,6 +328,7 @@ def _prepare_experiment(
         app_inst.set_formatted_executables(context.formatted_executables)
 
         if app_inst.package_manager is not None:
+            app_inst.package_manager.define_missing_packages(self._workspace)
             app_inst.define_variable(
                 self.keywords.env_path,
                 os.path.join(

lib/ramble/ramble/language/shared_language.py

@@ -161,7 +161,14 @@ def _execute_figure_of_merit(obj):
 
 @shared_directive("compilers")
 def define_compiler(
-    name, pkg_spec, compiler_spec=None, compiler=None, package_manager=None, when=None, **kwargs
+    name,
+    pkg_spec,
+    compiler_spec=None,
+    compiler=None,
+    package_manager=None,
+    inject_if_missing=False,
+    when=None,
+    **kwargs,
 ):
     """Defines the compiler that will be used with this object
 
@@ -175,6 +182,8 @@ def define_compiler(
         compiler (str): Package name to use for compilation
         package_manager (str): Glob supported pattern to match package managers
                                this compiler applies to
+        inject_if_missing (bool): Whether the package should be defined if a
+                                  matching package is not already defined
         when (list | None): List of when conditions to apply to directive
     """
 
@@ -195,7 +204,12 @@ def _execute_define_compiler(obj):
 
         obj.compilers[name].append(
             SoftwareSpec(
-                name, pkg_spec, compiler=compiler, compiler_spec=compiler_spec, when=when_list
+                name,
+                pkg_spec,
+                compiler=compiler,
+                compiler_spec=compiler_spec,
+                inject_if_missing=inject_if_missing,
+                when=when_list,
             )
         )
 
@@ -204,7 +218,14 @@ def _execute_define_compiler(obj):
 
 @shared_directive("software_specs")
 def software_spec(
-    name, pkg_spec, compiler_spec=None, compiler=None, package_manager=None, when=None, **kwargs
+    name,
+    pkg_spec,
+    compiler_spec=None,
+    compiler=None,
+    package_manager=None,
+    inject_if_missing=False,
+    when=None,
+    **kwargs,
 ):
     """Defines a new software spec needed for this object.
 
@@ -223,6 +244,8 @@ def software_spec(
         compiler (str): Package name to use as compiler for compiling this package
         package_manager (str): Glob supported pattern to match package managers
                                this package applies to
+        inject_if_missing (bool): Whether the package should be added to experiment
+                                  environments automatically or not.
         when (list | None): List of when conditions to apply to directive
     """
 
@@ -244,7 +267,12 @@ def _execute_software_spec(obj):
         # Define the spec
         obj.software_specs[name].append(
             SoftwareSpec(
-                name, pkg_spec, compiler=compiler, compiler_spec=compiler_spec, when=when_list
+                name,
+                pkg_spec,
+                compiler=compiler,
+                compiler_spec=compiler_spec,
+                inject_if_missing=inject_if_missing,
+                when=when_list,
             )
         )
 

lib/ramble/ramble/software_environments.py

@@ -14,6 +14,7 @@
 from ramble.expander import Expander
 from ramble.namespace import namespace
 from ramble.util.logger import logger
+from ramble.util.spec_utils import SoftwareSpec
 
 SUB_INDENT = 2
 
@@ -55,6 +56,7 @@ def __init__(
         self.pkg_info = pkg_info
         self._package_type = "Base"
         self._used = False
+        self.injected = False
 
     def mark_used(self):
         """Mark this package a used"""
@@ -107,7 +109,9 @@ def info(
         indentation = " " * indent
         color = rucolor.level_func(color_level)
 
-        out_str = color(f"{indentation}{self._package_type} package: {self.name}\n")
+        injected_str = "" if not self.injected else " (injected by ramble)"
+
+        out_str = color(f"{indentation}{self._package_type} package: {self.name} {injected_str}\n")
         return out_str
 
     def __str__(self):
@@ -466,14 +470,22 @@ def __eq__(self, other):
         Returns:
             (bool): True if environments are equivalent, False otherwise
         """
-        equal = self.name == other.name and len(self._packages) == len(other._packages)
-
-        if not equal:
+        if not self.name == other.name:
             return False
 
-        for self_pkg, other_pkg in zip(self._packages, other._packages):
-            if self_pkg != other_pkg:
-                return False
+        self_pkgs = {}
+        other_pkgs = {}
+
+        for self_pkg in self._packages:
+            if not self_pkg.injected:
+                self_pkgs[self_pkg.name] = self_pkg
+
+        for other_pkg in other._packages:
+            if not other_pkg.injected:
+                other_pkgs[other_pkg.name] = other_pkg
+
+        if self_pkgs != other_pkgs:
+            return False
 
         return True
 
@@ -853,6 +865,7 @@ def define_compiler_packages(self, environment: RenderedEnvironment, expander: E
 
                             if rendered_pkg.compiler:
                                 cur_compiler = rendered_pkg.compiler
+
                     if not added:
                         raise RambleSoftwareEnvironmentError(
                             f"Compiler {pkg.compiler} used, but not "
@@ -921,6 +934,45 @@ def package_specs_for_environment(self, environment: SoftwareEnvironment):
         for pkg in environment._packages:
             yield pkg.spec_str(all_packages=self._rendered_packages, compiler=False)
 
+    def add_spec_to_environment(
+        self,
+        environment: SoftwareEnvironment,
+        spec: SoftwareSpec,
+        expander: Expander,
+        package_manager,
+    ):
+        """Add a spec to a given environment
+
+        Creates a new template / rendered package (if needed) from the input spec,
+        and adds to the template and rendered environment as a package in the environment.
+
+        Args:
+            environment (SoftwareEnvironment): Rendered environment to add package to
+            spec (ramble.util.spec_utils.SoftwareSpec): Software spec to add to environment
+            expander (ramble.expander.Expander): Experiment's expander object,
+                                                 to render package / environment with
+            package_manager: Package manager from the experiment
+        """
+
+        pm_name = package_manager.spec_prefix
+
+        if spec.name not in self._package_templates:
+            template_package = TemplatePackage(spec.name, spec.to_dict())
+
+            self._package_templates[spec.name] = template_package
+
+        template_package = self._package_templates[spec.name]
+
+        rendered_package = template_package.render_package(expander, package_manager)
+        rendered_package.injected = True
+
+        if rendered_package.name not in self._rendered_packages:
+            template_package.add_rendered_package(
+                rendered_package, self._rendered_packages, pm_name
+            )
+
+        environment.add_package(rendered_package)
+
     def _check_environment(self, environment):
         """Check an environment for common issues