Merge pull request #1064 from dapomeroy/executable_when

Add `when` support to executable directive

Author: douglasjacobsen

lib/ramble/ramble/application.py

@@ -963,11 +963,27 @@ def _check_object_validators(self):
                         else:
                             logger.warn(err_msg)
 
+    def _get_filtered_executables(self) -> dict:
+        """Returns a dict of executables that satisfy `when` conditions"""
+        filtered_executables = {}
+        all_executables = self.executables.copy()
+        for when_set, executables in all_executables.items():
+            if self.expander.satisfies(when_set, variant_set=self.object_variants):
+                for executable in executables:
+                    if executable in filtered_executables:
+                        logger.die(
+                            f"Executable {executable} is defined for overlapping `when` "
+                            "conditions. Ensure conditions are mutually exclusive."
+                        )
+                filtered_executables.update(executables)
+
+        return filtered_executables
+
     def _define_custom_executables(self):
         # Define custom executables
         if namespace.custom_executables in self.internals:
             for name, conf in self.internals[namespace.custom_executables].items():
-                if name in self.executables or name in self.custom_executables:
+                if name in self._get_filtered_executables() or name in self.custom_executables:
                     experiment_namespace = self.expander.expand_var_name("experiment_namespace")
                     raise ExecutableNameError(
                         f"In experiment {experiment_namespace} "
@@ -996,11 +1012,18 @@ def _get_executable_graph(self, workload_name):
                     builtin_objects.append(obj)
                     all_builtins.append(builtins)
 
-        all_executables = self.executables.copy()
-        all_executables.update(self.custom_executables)
+        filtered_executables = self._get_filtered_executables()
+        filtered_executables.update(self.custom_executables)
+
+        filtered_exec_order = []
+        for executable in exec_order:
+            if executable in filtered_executables or any(executable in b for b in all_builtins):
+                filtered_exec_order.append(executable)
+            else:
+                logger.debug(f"Skipping executable {executable}. `When` conditions not satisfied.")
 
         executable_graph = ramble.graphs.ExecutableGraph(
-            exec_order, all_executables, builtin_objects, all_builtins, self
+            filtered_exec_order, filtered_executables, builtin_objects, all_builtins, self
         )
 
         # Perform executable injection
@@ -1693,9 +1716,10 @@ def _archive_experiments(self, workspace, app_inst=None):
             # Copy all log files from executables
             exec_logs = set()
             workload = self.workloads[self.expander.workload_name]
+            filtered_executables = self._get_filtered_executables()
             for exec_name in workload.executables:
-                if exec_name in self.executables:
-                    exec_obj = self.executables[exec_name]
+                if exec_name in filtered_executables:
+                    exec_obj = filtered_executables[exec_name]
                     exec_log = self.expander.expand_var(exec_obj.redirect)
                     exec_logs.add(exec_log)
 

lib/ramble/ramble/language/application_language.py

@@ -117,7 +117,7 @@ def _execute_workload_groups(app):
 
 
 @application_directive("executables")
-def executable(name, template, **kwargs):
+def executable(name, template, when=None, **kwargs):
     """Adds an executable to this application
 
     Defines a new executable that can be used to configure workloads and
@@ -141,12 +141,18 @@ def executable(name, template, **kwargs):
                               both) to capture. Defaults to stdout
         run_in_background (bool): Optional, Declare if the command should
                                      run in the background. Defaults to False
+        when (list | None): List of when conditions to apply to directive
     """
 
     def _execute_executable(app):
         from ramble.util.executable import CommandExecutable
 
-        app.executables[name] = CommandExecutable(name=name, template=template, **kwargs)
+        when_list = ramble.language.language_helpers.build_when_list(when, app, name, "executable")
+        when_set = frozenset(when_list)
+        if when_set not in app.executables:
+            app.executables[when_set] = {}
+
+        app.executables[when_set][name] = CommandExecutable(name=name, template=template, **kwargs)
 
     return _execute_executable
 

lib/ramble/ramble/language/language_base.py

@@ -238,6 +238,7 @@ def _wrapper(*args, **_kwargs):
                         "modifier_variable",
                         "package_manager_variable",
                         "workflow_manager_variable",
+                        "executable",
                     ]:
                         msg = (
                             'directive "{0}" cannot be used within a "when"'

lib/ramble/ramble/language/language_helpers.py

@@ -7,7 +7,8 @@
 # except according to those terms.
 
 import fnmatch
-from typing import Any, List
+from collections import OrderedDict
+from typing import Any, List, Union
 
 from ramble.language.language_base import DirectiveError
 
@@ -73,16 +74,17 @@ def merge_definitions(
         single_type, multiple_type, single_arg_name, multiple_arg_name, directive_name
     )
 
-    all_types = []
+    merged_types = []
 
     if single_type:
-        all_types.append(single_type)
+        merged_types.append(single_type)
 
     if multiple_type:
-        expanded_multiple_type = expand_patterns(multiple_type, multiple_pattern_match)
-        all_types.extend(expanded_multiple_type)
+        merged_types.extend(multiple_type)
 
-    return all_types
+    merged_types_expanded = expand_patterns(merged_types, multiple_pattern_match)
+
+    return merged_types_expanded
 
 
 def require_definition(
@@ -128,32 +130,52 @@ def require_definition(
     )
 
 
-def expand_patterns(multiple_type: list, multiple_pattern_match: list):
+def expand_patterns(merged_types: list, multiple_pattern_match: Union[list, dict]):
     """Expand wildcard patterns within a list of names
 
     This method takes an input list containing wildcard patterns and expands the
     wildcard with values matching a list of names. Returns a list containing
     matching names and any inputs with zero matches.
 
+    If multiple_pattern_match is a dict keyed on 'when', it checks the input
+    against patterns in all 'when' conditions, without evaluating them, and
+    returns a list containing names that match under any when condition, and
+    any inputs with zero matches.
+
     Args:
-        multiple_types: List of strings for type names, may contain wildcards
-        multiple_pattern_match: List of strings to match against patterns in multiple_type
+        merged_types: List of strings for type names, may contain wildcards
+        multiple_pattern_match: List of strings (optional: nested in when_set
+            dict) to match against patterns in merged_types
 
     Returns:
         List of expanded patterns matching the names list plus patterns
-        not found in the names list.
+            not found in the names list.
     """
-
-    expanded_patterns = []
-    for input in multiple_type:
-        matched_inputs = fnmatch.filter(multiple_pattern_match, input)
-        if matched_inputs:
-            for matching_name in matched_inputs:
-                expanded_patterns.append(matching_name)
+    expanded_patterns = OrderedDict()
+    for input in merged_types:
+        expanded = False
+        if (
+            multiple_pattern_match
+            and isinstance(multiple_pattern_match, dict)
+            and isinstance(next(iter(multiple_pattern_match)), frozenset)
+        ):
+            for _, pattern_list in multiple_pattern_match.items():
+                matched_inputs = fnmatch.filter(pattern_list, input)
+                if matched_inputs:
+                    expanded = True
+                    for match in matched_inputs:
+                        expanded_patterns[match] = ""
         else:
-            expanded_patterns.append(input)
+            matched_inputs = fnmatch.filter(multiple_pattern_match, input)
+            if matched_inputs:
+                expanded = True
+                for match in matched_inputs:
+                    expanded_patterns[match] = ""
+
+        if not expanded:
+            expanded_patterns[input] = ""
 
-    return expanded_patterns
+    return list(expanded_patterns.keys())
 
 
 def build_when_list(

lib/ramble/ramble/test/application.py

@@ -16,6 +16,8 @@
     "mutable_config", "mutable_mock_workspace_path", "mutable_mock_apps_repo"
 )
 
+_FS = frozenset()
+
 
 def basic_exp_dict():
     """To set expander consistently with test_wl2 of builtin.mock/applications/basic"""
@@ -68,15 +70,15 @@ def test_basic_app(mutable_mock_apps_repo):
     assert len(basic_inst.workloads["test_wl"].executables) == 1
     foo_exec = basic_inst.workloads["test_wl"].find_executable("foo")
     assert foo_exec is not None
-    foo_exec = basic_inst.executables[foo_exec]
+    foo_exec = basic_inst.executables[_FS][foo_exec]
     assert foo_exec.template == ["bar"]
     assert not foo_exec.mpi
 
     assert len(basic_inst.workloads["test_wl"].inputs) == 1
     example_input = basic_inst.workloads["test_wl"].find_input("input")
     assert example_input is not None
 
-    assert len(basic_inst.workloads["test_wl"].variables[frozenset()]) == 2
+    assert len(basic_inst.workloads["test_wl"].variables[_FS]) == 2
     possible_vars = basic_inst.workloads["test_wl"].find_variable("my_var")
     assert len(possible_vars) == 1
     assert possible_vars[0].default == "1.0"
@@ -86,7 +88,7 @@ def test_basic_app(mutable_mock_apps_repo):
     assert len(basic_inst.workloads["test_wl2"].executables) == 1
     bar_exec = basic_inst.workloads["test_wl2"].find_executable("bar")
     assert bar_exec is not None
-    bar_exec = basic_inst.executables[bar_exec]
+    bar_exec = basic_inst.executables[_FS][bar_exec]
     assert bar_exec.template == ["baz"]
     assert bar_exec.mpi
 
@@ -104,8 +106,8 @@ def test_basic_app(mutable_mock_apps_repo):
     assert exec_graph.get_node("bar") is not None
     assert exec_graph.get_node("builtin::env_vars") is not None
 
-    assert "test_fom" in basic_inst.figures_of_merit[frozenset()][frozenset()]
-    fom_conf = basic_inst.figures_of_merit[frozenset()][frozenset()]["test_fom"]
+    assert "test_fom" in basic_inst.figures_of_merit[_FS][_FS]
+    fom_conf = basic_inst.figures_of_merit[_FS][_FS]["test_fom"]
     assert fom_conf["log_file"] == "{log_file}"
     assert fom_conf["regex"] == r"(?P<test>[0-9]+\.[0-9]+).*seconds.*"  # noqa: W605
     assert fom_conf["group_name"] == "test"
@@ -192,7 +194,7 @@ def test_required_builtins(mutable_mock_apps_repo, app):
     app_inst.define_variable("application_name", app)
 
     required_builtins = []
-    for builtin, conf in app_inst.builtins[frozenset()].items():
+    for builtin, conf in app_inst.builtins[_FS].items():
         if conf[app_inst._builtin_required_key]:
             required_builtins.append(builtin)
 
@@ -211,7 +213,7 @@ def test_register_builtin_app(mutable_mock_apps_repo):
 
     required_builtins = []
     excluded_builtins = []
-    for builtin, conf in app_inst.builtins[frozenset()].items():
+    for builtin, conf in app_inst.builtins[_FS].items():
         if conf[app_inst._builtin_required_key]:
             required_builtins.append(builtin)
         else:

lib/ramble/ramble/test/application_inheritance.py

@@ -8,19 +8,21 @@
 
 from ramble.test.application import basic_exp_dict
 
+_FS = frozenset()
+
 
 def test_basic_inheritance(mutable_mock_apps_repo):
     app_inst = mutable_mock_apps_repo.get("basic-inherited")
     exp_dict = basic_exp_dict()
     app_inst.set_variables(exp_dict, None)
     app_inst.define_variable("application_name", "basic-inherited")
 
-    assert "foo" in app_inst.executables
-    assert app_inst.executables["foo"].template == ["bar"]
-    assert not app_inst.executables["foo"].mpi
-    assert "bar" in app_inst.executables
-    assert app_inst.executables["bar"].template == ["baz"]
-    assert app_inst.executables["bar"].mpi
+    assert "foo" in app_inst.executables[_FS]
+    assert app_inst.executables[_FS]["foo"].template == ["bar"]
+    assert not app_inst.executables[_FS]["foo"].mpi
+    assert "bar" in app_inst.executables[_FS]
+    assert app_inst.executables[_FS]["bar"].template == ["baz"]
+    assert app_inst.executables[_FS]["bar"].mpi
 
     assert "test_wl" in app_inst.workloads
     assert app_inst.workloads["test_wl"].executables == ["foo"]
@@ -49,8 +51,8 @@ def test_basic_inheritance(mutable_mock_apps_repo):
     assert exec_graph.get_node("foo") is not None
     assert exec_graph.get_node("builtin::env_vars") is not None
 
-    assert "test_fom" in app_inst.figures_of_merit[frozenset()][frozenset()]
-    fom_conf = app_inst.figures_of_merit[frozenset()][frozenset()]["test_fom"]
+    assert "test_fom" in app_inst.figures_of_merit[_FS][_FS]
+    fom_conf = app_inst.figures_of_merit[_FS][_FS]["test_fom"]
     assert fom_conf["log_file"] == "{log_file}"
     assert fom_conf["regex"] == r"(?P<test>[0-9]+\.[0-9]+).*seconds.*"  # noqa: W605
     assert fom_conf["group_name"] == "test"

lib/ramble/ramble/test/application_language.py

@@ -17,6 +17,8 @@
     ExecutableApplication,  # noqa: F405
 ]
 
+_FS = frozenset()
+
 
 @deprecation.fail_if_not_removed
 @pytest.mark.parametrize("app_class", app_types)
@@ -257,18 +259,16 @@ def test_executable_directive(app_class):
 
     assert hasattr(app_inst, "executables")
     for exe_name, conf in test_defs.items():
-        assert exe_name in app_inst.executables
+        assert exe_name in app_inst.executables[_FS]
         for conf_name, conf_val in conf.items():
-            assert hasattr(app_inst.executables[exe_name], conf_name)
-            assert conf_val == getattr(app_inst.executables[exe_name], conf_name)
+            assert hasattr(app_inst.executables[_FS][exe_name], conf_name)
+            assert conf_val == getattr(app_inst.executables[_FS][exe_name], conf_name)
 
 
 @pytest.mark.parametrize("app_class", app_types)
 def test_figure_of_merit_directive(app_class):
     test_defs = {}
 
-    _FS = frozenset()
-
     app_inst = app_class("/not/a/path")
     test_defs.update(add_figure_of_merit(app_inst))