Merge pull request #4742 from mnlevy1981/add_postprocessing_env_file

Create env_postprocessing.xml in case root

Author: jedwards4b

.github/workflows/testing.yml

@@ -153,7 +153,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Cache inputdata
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /storage/inputdata
           key: inputdata-2

CIME/XML/env_postprocessing.py

@@ -0,0 +1,22 @@
+"""
+Interface to the env_postprocessing.xml file.  This class inherits from EnvBase
+"""
+from CIME.XML.standard_module_setup import *
+
+from CIME.XML.env_base import EnvBase
+
+from CIME import utils
+
+logger = logging.getLogger(__name__)
+
+
+class EnvPostprocessing(EnvBase):
+    def __init__(
+        self, case_root=None, infile="env_postprocessing.xml", read_only=False
+    ):
+        """
+        initialize an object interface to file env_postprocessing.xml in the case directory
+        """
+        schema = os.path.join(utils.get_schema_path(), "env_entry_id.xsd")
+
+        EnvBase.__init__(self, case_root, infile, schema=schema, read_only=read_only)

CIME/XML/postprocessing.py

@@ -0,0 +1,40 @@
+"""
+Interface to the config_postprocessing.xml file.  This class inherits from EntryID
+"""
+
+from CIME.XML.standard_module_setup import *
+from CIME.XML.entry_id import EntryID
+from CIME.XML.files import Files
+from CIME.utils import expect
+
+logger = logging.getLogger(__name__)
+
+
+class Postprocessing(EntryID):
+    def __init__(self, infile=None, files=None):
+        """
+        initialize an object
+        """
+        if files is None:
+            files = Files()
+        if infile is None:
+            infile = files.get_value("POSTPROCESSING_SPEC_FILE")
+        if infile is not None:
+            self.file_exists = os.path.isfile(infile)
+        else:
+            self.file_exists = False
+        if not self.file_exists:
+            return
+        expect(infile, "No postprocessing file defined in {}".format(files.filename))
+
+        schema = files.get_schema("POSTPROCESSING_SPEC_FILE")
+
+        EntryID.__init__(self, infile, schema=schema)
+
+        # Append the contents of $HOME/.cime/config_postprocessing.xml if it exists
+        # This could cause problems if node matchs are repeated when only one is expected
+        infile = os.path.join(
+            os.environ.get("HOME"), ".cime", "config_postprocessing.xml"
+        )
+        if os.path.exists(infile):
+            EntryID.read(self, infile)

CIME/case/case.py

@@ -29,6 +29,7 @@
 from CIME.XML.grids import Grids
 from CIME.XML.batch import Batch
 from CIME.XML.workflow import Workflow
+from CIME.XML.postprocessing import Postprocessing
 from CIME.XML.pio import PIO
 from CIME.XML.archive import Archive
 from CIME.XML.env_test import EnvTest
@@ -40,6 +41,7 @@
 from CIME.XML.env_archive import EnvArchive
 from CIME.XML.env_batch import EnvBatch
 from CIME.XML.env_workflow import EnvWorkflow
+from CIME.XML.env_postprocessing import EnvPostprocessing
 from CIME.XML.generic_xml import GenericXML
 from CIME.user_mod_support import apply_user_mods
 from CIME.aprun import get_aprun_cmd_for_case
@@ -109,6 +111,7 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False
                 case_root
             ),
         )
+        self._existing_case = os.path.isdir(case_root)
 
         self._caseroot = case_root
         logger.debug("Initializing Case.")
@@ -356,6 +359,10 @@ def read_xml(self):
         self._env_entryid_files.append(
             EnvWorkflow(self._caseroot, read_only=self._force_read_only)
         )
+        if not self._existing_case or os.path.isfile("env_postprocessing.xml"):
+            self._env_entryid_files.append(
+                EnvPostprocessing(self._caseroot, read_only=self._force_read_only)
+            )
 
         if os.path.isfile(os.path.join(self._caseroot, "env_test.xml")):
             self._env_entryid_files.append(
@@ -430,6 +437,19 @@ def flush(self, flushall=False):
             # do not flush if caseroot wasnt created
             return
 
+        _postprocessing_spec_file = self.get_value("POSTPROCESSING_SPEC_FILE")
+        if _postprocessing_spec_file is not None:
+            have_postprocessing = os.path.isfile(_postprocessing_spec_file)
+        else:
+            have_postprocessing = False
+        if not have_postprocessing:
+            # Remove env_postprocessing.xml from self._files
+            self._files = [
+                file
+                for file in self._files
+                if file.get_id() != "env_postprocessing.xml"
+            ]
+
         for env_file in self._files:
             env_file.write(force_write=flushall)
 
@@ -1577,6 +1597,11 @@ def configure(
 
         workflow = Workflow(files=files)
 
+        postprocessing = Postprocessing(files=files)
+        if postprocessing.file_exists:
+            env_postprocessing = self.get_env("postprocessing")
+            env_postprocessing.add_elements_by_group(srcobj=postprocessing)
+
         env_batch.set_batch_system(batch, batch_system_type=batch_system_type)
 
         bjobs = workflow.get_workflow_jobs(machine=machine_name, workflowid=workflowid)
@@ -2216,6 +2241,8 @@ def set_file(self, xmlfile):
                     new_env_file = EnvBatch(infile=xmlfile)
                 elif ftype == "env_workflow.xml":
                     new_env_file = EnvWorkflow(infile=xmlfile)
+                elif ftype == "env_postprocessing.xml":
+                    new_env_file = EnvPostprocessing(infile=xmlfile)
                 elif ftype == "env_test.xml":
                     new_env_file = EnvTest(infile=xmlfile)
                 elif ftype == "env_archive.xml":

CIME/data/config/cesm/config_files.xml

@@ -55,6 +55,15 @@
     <schema>$CIMEROOT/CIME/data/config/xml_schemas/config_batch.xsd</schema>
   </entry>
 
+  <entry id="POSTPROCESSING_SPEC_FILE">
+    <type>char</type>
+    <default_value>$SRCROOT/tools/CUPiD/cime_config/config_tool.xml</default_value>
+    <group>case_last</group>
+    <file>env_case.xml</file>
+    <desc>file containing postprocessing XML configuration (for documentation only - DO NOT EDIT)</desc>
+    <schema>$CIMEROOT/CIME/data/config/xml_schemas/entry_id.xsd</schema>
+  </entry>
+
   <entry id="WORKFLOW_SPEC_FILE">
     <type>char</type>
     <default_value>$SRCROOT/ccs_config/machines/config_workflow.xml</default_value>

CIME/data/config/config_headers.xml

@@ -17,6 +17,13 @@
     </header>
   </file>
 
+  <file name="env_postprocessing.xml">
+    <header>
+      These variables may be changed anytime during a run, they
+      control jobs that are dependent on case.run.
+    </header>
+  </file>
+
   <file name="env_case.xml">
     <header>
     These variables CANNOT BE CHANGED once a case has been created.