Made some progress

Author: hwikle-lanl

lib/pavilion/resolver/proto_test.py

@@ -329,6 +329,7 @@ def _permute_sched(self, var_man, used_per_vars) -> List[variables.VariableSetMa
 
         sched_name = self.config['scheduler']
         sched = schedulers.get_plugin(sched_name)
+        test_name = self.config.get('name', '<no name>')
 
         # Resolve the variables that don't depend on sched, but have complicated relationships
         # we couldn't solve iteratively. If there's anything left at this point, it will

lib/pavilion/schedulers/config.py

@@ -261,10 +261,6 @@ def check_leaves(cls, elem):
             raise ValueError(elem)
 
 
-class SchedConfigError(ValueError):
-    """Raised when there's a problem with the scheduler configuration."""
-
-
 def min_int(name, min_val, required=True):
     """Return a callback that ensures the argument >= min_val. If not required,
     an empty value will return None."""
@@ -567,7 +563,7 @@ def _validate_config(config: Dict[str, str],
 
             except ValueError as err:
                 raise SchedConfigError("Config value for key '{}' had a validation "
-                                       "error.".format(key), err)
+                                       "error.".format(key), prior_error=err)
 
         elif isinstance(validator, (tuple, list)):
             if value not in validator:

lib/pavilion/scriptcomposer.py

@@ -132,7 +132,11 @@ def module_change(self, module, sys_vars, config_wrappers):
     def module_purge(self) -> None:
         """Add a module purge to the script."""
 
-        self._script_lines.append("module purge")
+        self._script_lines.extend([
+            "if which module; then",
+            "\tmodule purge",
+            "fi"
+            ])
 
     def newline(self):
         """Function that just adds a newline to the script lines."""

lib/pavilion/series/series.py

@@ -418,7 +418,7 @@ def check_cancelled(self) -> bool:
         return False
 
     def run(self, build_only: bool = False, rebuild: bool = False,
-            local_builds_only: bool = False, purge: bool = True):
+            local_builds_only: bool = False):
         """Build and kickoff all of the test sets in the series.
 
         :param build_only: Only build the tests, do not run them.
@@ -471,7 +471,7 @@ def run(self, build_only: bool = False, rebuild: bool = False,
 
                 try:
                     self._run_set(test_set, build_only=build_only,
-                                  rebuild=rebuild, local_builds_only=local_builds_only, purge=purge)
+                                  rebuild=rebuild, local_builds_only=local_builds_only)
                 except TestSetError as err:
                     self.status.set(SERIES_STATES.ERROR,
                                     "Error running test set {}. See the series log "
@@ -506,16 +506,15 @@ def run(self, build_only: bool = False, rebuild: bool = False,
         # Completion will be set when looked for.
 
 
-    def _run_set(self, test_set: TestSet, build_only: bool, rebuild: bool, local_builds_only: bool,
-                 purge: bool):
+    def _run_set(self, test_set: TestSet, build_only: bool, rebuild: bool, local_builds_only: bool):
         """Run all requested tests in the given test set."""
 
         # Track which builds we've already marked as deprecated, when doing rebuilds.
         deprecated_builds = set()
         failed_builds = dict()
         tests_running = 0
 
-        for test_batch in test_set.make_iter(build_only, rebuild, local_builds_only, purge):
+        for test_batch in test_set.make_iter(build_only, rebuild, local_builds_only):
 
             # Add all the tests we created to this test set.
             self._add_tests(test_batch, test_set.iter_name)

lib/pavilion/series/test_set.py

@@ -189,7 +189,7 @@ def __ordered_split(self) -> List['TestSet']:
 
         return test_sets
 
-    def make_iter(self, build_only=False, rebuild=False, local_builds_only=False, purge=True) \
+    def make_iter(self, build_only=False, rebuild=False, local_builds_only=False) \
                   -> Iterator[List[TestRun]]:
         """Resolve the given tests names and options into actual test run objects, and print
         the test creation status.  This returns an iterator over batches tests, respecting the
@@ -277,7 +277,7 @@ def make_iter(self, build_only=False, rebuild=False, local_builds_only=False, pu
                 try:
                     test_run = TestRun(pav_cfg=self.pav_cfg, config=ptest.config,
                                        var_man=ptest.var_man, rebuild=rebuild,
-                                       build_only=build_only, purge=purge)
+                                       build_only=build_only)
                     if not test_run.skipped:
                         test_run.save()
                         self.tests.append(test_run)
@@ -339,12 +339,12 @@ def make_iter(self, build_only=False, rebuild=False, local_builds_only=False, pu
         self.all_tests_made = True
 
 
-    def make(self, build_only=False, rebuild=False, local_builds_only=False, purge=True):
+    def make(self, build_only=False, rebuild=False, local_builds_only=False):
         """As per make_iter(), but create all of the tests. This doesn't
         respect batch sizes, etc, and is entirely for simplifying unit testing."""
 
         all_tests = []
-        for test_batch in self.make_iter(build_only, rebuild, local_builds_only, purge):
+        for test_batch in self.make_iter(build_only, rebuild, local_builds_only):
             all_tests.extend(test_batch)
 
         return all_tests

lib/pavilion/test_run/test_run.py

@@ -101,8 +101,7 @@ class TestRun(TestAttributes):
     """Directory that holds build templates."""
 
     def __init__(self, pav_cfg: PavConfig, config: Dict, var_man: VariableSetManager = None,
-                 _id: int = None, rebuild: bool = False, build_only: bool = False,
-                 purge: bool = True):
+                 _id: int = None, rebuild: bool = False, build_only: bool = False):
         """Create an new TestRun object. If loading an existing test
     instance, use the ``TestRun.from_id()`` method.
 
@@ -111,7 +110,6 @@ def __init__(self, pav_cfg: PavConfig, config: Dict, var_man: VariableSetManager
     :param bool build_only: Only build this test run, do not run it.
     :param bool rebuild: After determining the build name, deprecate it and
         select a new, non-deprecated build.
-    :param bool purge: Whether or not to perform a module purge before running/building
     :param int _id: The test id of an existing test. (You should be using
         TestRun.load)."""
 
@@ -152,7 +150,6 @@ def __init__(self, pav_cfg: PavConfig, config: Dict, var_man: VariableSetManager
             # Set basic attributes
             self.id = id_tmp  # pylint: disable=invalid-name
             self.build_only = build_only
-            self.purge = purge
             self._complete = False
             self.created = time.time()
             self.name = self.make_name(config)
@@ -1151,7 +1148,9 @@ def _write_script(self, stype: str, path: Path, config: dict, module_wrappers: d
 
         script.command(f'echo "(pav) Setting up {stype} environment."')
 
-        if self.purge:
+        purge = config.get(stype, {}).get("purge_modules")
+
+        if purge:
             script.newline()
             script.comment("Start with a fresh environment")
             script.module_purge()

lib/pavilion/unittest.py

@@ -313,7 +313,7 @@ def _load_test(self, name: str, platform: str = 'this', host: str = 'this',
     del __config_lines
 
     def _quick_test(self, cfg=None, name="quick_test",
-                    build=True, finalize=True, purge=True):
+                    build=True, finalize=True):
         """Create a test run object to work with.
         The default is a simple hello world test with the raw scheduler.
 
@@ -347,7 +347,7 @@ def _quick_test(self, cfg=None, name="quick_test",
 
         cfg = resolve.test_config(cfg, var_man)
 
-        test = TestRun(pav_cfg=self.pav_cfg, config=cfg, var_man=var_man, purge=purge)
+        test = TestRun(pav_cfg=self.pav_cfg, config=cfg, var_man=var_man)
         if test.skipped:
             # You can't proceed further with a skipped test.
             return test

test/tests/mod_wrapper_tests.py

@@ -311,21 +311,57 @@ def check_test(ctest, expected_lines):
 
     @unittest.skipIf(not has_module_cmd() and find_module_init() is None,
                      "Could not find a module system.")
-    def test_module_purge(self):
-        """Test that a module purge is performed when building and running tests."""
+    def test_run_module_purge(self):
+        """Test that a module purge is performed when running tests."""
 
+        # Test that a module purge is performed by default
         test_cfg = self._quick_test_cfg()
         test_cfg['run']['cmds'] = [
             '[[ $(module -t list 2>&1) = "No modules loaded" ]] || exit 1',
         ]
         test_cfg['run']['preamble'].append('module load test_mod1 || exit 2')
 
-        test = self._quick_test(test_cfg, purge=False)
+        test = self._quick_test(test_cfg)
         run_result = test.run()
 
-        self.assertEqual(run_result, 1)
+        self.assertEqual(run_result, 0)
 
+        # Check that we can disable purging
+        test_cfg = self._quick_test_cfg()
+        test_cfg['run']['cmds'] = [
+            '[[ $(module -t list 2>&1) = "No modules loaded" ]] || exit 1',
+        ]
+        test_cfg['run']['preamble'].append('module load test_mod1 || exit 2')
+        test_config["run"]["purge_modules"] = False
         test = self._quick_test(test_cfg)
         run_result = test.run()
 
-        self.assertEqual(run_result, 0)
+        self.assertEqual(run_result, 1)
+
+    def test_build_module_purge(self):
+        """Test that a module purge is performed when building tests."""
+
+        # Test that a module purge is performed by default
+        test_cfg = self._quick_test_cfg()
+        test_cfg['build']['cmds'] = [
+            "# This comment exists so the test hashes to a distinct value"
+            '[[ $(module -t list 2>&1) = "No modules loaded" ]] || exit 1',
+        ]
+        test_cfg['run']['preamble'].append('module load test_mod1 || exit 2')
+
+        test = self._quick_test(test_cfg)
+        build_result = test.build()
+
+        self.assertEqual(build_result, 0)
+
+        # Check that we can disable purging
+        test_cfg = self._quick_test_cfg()
+        test_cfg['build']['cmds'] = [
+            '[[ $(module -t list 2>&1) = "No modules loaded" ]] || exit 1',
+        ]
+        test_cfg['build']['preamble'].append('module load test_mod1 || exit 2')
+        test_cfg["build"]["purge_modules"] = False
+        test = self._quick_test(test_cfg)
+        build_result = test.build()
+
+        self.assertEqual(build_result, 1)