From 80102adf780bfb18f12e5a7d54a8d25d8bc0eb4a Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 30 Jan 2025 14:18:59 -0700 Subject: [PATCH 1/9] Create env_postprocessing.xml in case root The contents of this new file are based on a config file in CUPiD (cime_config/config_tool.xml) that is comparable to a component's config_component.xml file --- CIME/XML/env_postprocessing.py | 23 ++++++++++++++++++ CIME/XML/postprocessing.py | 32 ++++++++++++++++++++++++++ CIME/case/case.py | 11 +++++++++ CIME/data/config/cesm/config_files.xml | 9 ++++++++ CIME/data/config/config_headers.xml | 7 ++++++ 5 files changed, 82 insertions(+) create mode 100644 CIME/XML/env_postprocessing.py create mode 100644 CIME/XML/postprocessing.py diff --git a/CIME/XML/env_postprocessing.py b/CIME/XML/env_postprocessing.py new file mode 100644 index 00000000000..0eb98ffbba0 --- /dev/null +++ b/CIME/XML/env_postprocessing.py @@ -0,0 +1,23 @@ +""" +Interface to the env_postprocessing.xml file. This class inherits from EnvBase +""" +from CIME.XML.standard_module_setup import * + +from CIME.XML.env_base import EnvBase + +from CIME import utils +from CIME.utils import convert_to_type + +logger = logging.getLogger(__name__) + + +class EnvPostprocessing(EnvBase): + def __init__( + self, case_root=None, infile="env_postprocessing.xml", read_only=False + ): + """ + initialize an object interface to file env_postprocessing.xml in the case directory + """ + schema = os.path.join(utils.get_schema_path(), "env_entry_id.xsd") + + EnvBase.__init__(self, case_root, infile, schema=schema, read_only=read_only) diff --git a/CIME/XML/postprocessing.py b/CIME/XML/postprocessing.py new file mode 100644 index 00000000000..08233c7bf8c --- /dev/null +++ b/CIME/XML/postprocessing.py @@ -0,0 +1,32 @@ +""" +Interface to the config_postprocessing.xml file. This class inherits from EntryID +""" + +from CIME.XML.standard_module_setup import * +from CIME.XML.entry_id import EntryID +from CIME.XML.files import Files +from CIME.utils import expect + +logger = logging.getLogger(__name__) + + +class Postprocessing(EntryID): + def __init__(self, infile=None, files=None): + """ + initialize an object + """ + if files is None: + files = Files() + if infile is None: + infile = files.get_value("POSTPROCESSING_SPEC_FILE") + expect(infile, "No postprocessing file defined in {}".format(files.filename)) + + schema = files.get_schema("POSTPROCESSING_SPEC_FILE") + + EntryID.__init__(self, infile, schema=schema) + + # Append the contents of $HOME/.cime/config_postprocessing.xml if it exists + # This could cause problems if node matchs are repeated when only one is expected + infile = os.path.join(os.environ.get("HOME"), ".cime", "config_postprocessing.xml") + if os.path.exists(infile): + EntryID.read(self, infile) \ No newline at end of file diff --git a/CIME/case/case.py b/CIME/case/case.py index 4e2677c81fe..31959f54a68 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -29,6 +29,7 @@ from CIME.XML.grids import Grids from CIME.XML.batch import Batch from CIME.XML.workflow import Workflow +from CIME.XML.postprocessing import Postprocessing from CIME.XML.pio import PIO from CIME.XML.archive import Archive from CIME.XML.env_test import EnvTest @@ -40,6 +41,7 @@ from CIME.XML.env_archive import EnvArchive from CIME.XML.env_batch import EnvBatch from CIME.XML.env_workflow import EnvWorkflow +from CIME.XML.env_postprocessing import EnvPostprocessing from CIME.XML.generic_xml import GenericXML from CIME.user_mod_support import apply_user_mods from CIME.aprun import get_aprun_cmd_for_case @@ -356,6 +358,9 @@ def read_xml(self): self._env_entryid_files.append( EnvWorkflow(self._caseroot, read_only=self._force_read_only) ) + self._env_entryid_files.append( + EnvPostprocessing(self._caseroot, read_only=self._force_read_only) + ) if os.path.isfile(os.path.join(self._caseroot, "env_test.xml")): self._env_entryid_files.append( @@ -1577,6 +1582,10 @@ def configure( workflow = Workflow(files=files) + env_postprocessing = self.get_env("postprocessing") + postprocessing = Postprocessing(files=files) + env_postprocessing.add_elements_by_group(srcobj=postprocessing) + env_batch.set_batch_system(batch, batch_system_type=batch_system_type) bjobs = workflow.get_workflow_jobs(machine=machine_name, workflowid=workflowid) @@ -2216,6 +2225,8 @@ def set_file(self, xmlfile): new_env_file = EnvBatch(infile=xmlfile) elif ftype == "env_workflow.xml": new_env_file = EnvWorkflow(infile=xmlfile) + elif ftype == "env_postprocessing.xml": + new_env_file = EnvPostprocessing(infile=xmlfile) elif ftype == "env_test.xml": new_env_file = EnvTest(infile=xmlfile) elif ftype == "env_archive.xml": diff --git a/CIME/data/config/cesm/config_files.xml b/CIME/data/config/cesm/config_files.xml index ba7d3aaa240..3cc9eac40b2 100644 --- a/CIME/data/config/cesm/config_files.xml +++ b/CIME/data/config/cesm/config_files.xml @@ -55,6 +55,15 @@ $CIMEROOT/CIME/data/config/xml_schemas/config_batch.xsd + + char + $SRCROOT/tools/CUPiD/cime_config/config_tool.xml + case_last + env_case.xml + file containing postprocessing XML configuration (for documentation only - DO NOT EDIT) + $CIMEROOT/CIME/data/config/xml_schemas/entry_id.xsd + + char $SRCROOT/ccs_config/machines/config_workflow.xml diff --git a/CIME/data/config/config_headers.xml b/CIME/data/config/config_headers.xml index c0d939cf3a4..10f59f2c770 100644 --- a/CIME/data/config/config_headers.xml +++ b/CIME/data/config/config_headers.xml @@ -17,6 +17,13 @@ + +
+ These variables may be changed anytime during a run, they + control jobs that are dependent on case.run. +
+ +
These variables CANNOT BE CHANGED once a case has been created. From 1e9b23e9d8912c2feb44f93d4dc4ea87451bde97 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 28 Feb 2025 16:31:24 -0700 Subject: [PATCH 2/9] Add _file_exists to Postprocessing class If POSTPROCESSING_SPEC_FILE is not found, then we do not call env_postprocessing.add_elements_by_group(). This still creates an env_postprocessing.xml file that only contains
These variables may be changed anytime during a run, they control jobs that are dependent on case.run.
I think we'd ideally not create this file at all? Also, I ran pre-commit :) --- CIME/XML/env_postprocessing.py | 1 - CIME/XML/postprocessing.py | 9 +++++++-- CIME/case/case.py | 5 +++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CIME/XML/env_postprocessing.py b/CIME/XML/env_postprocessing.py index 0eb98ffbba0..90f56f24d64 100644 --- a/CIME/XML/env_postprocessing.py +++ b/CIME/XML/env_postprocessing.py @@ -6,7 +6,6 @@ from CIME.XML.env_base import EnvBase from CIME import utils -from CIME.utils import convert_to_type logger = logging.getLogger(__name__) diff --git a/CIME/XML/postprocessing.py b/CIME/XML/postprocessing.py index 08233c7bf8c..c52b499b5fd 100644 --- a/CIME/XML/postprocessing.py +++ b/CIME/XML/postprocessing.py @@ -19,6 +19,9 @@ def __init__(self, infile=None, files=None): files = Files() if infile is None: infile = files.get_value("POSTPROCESSING_SPEC_FILE") + self._file_exists = os.path.isfile(infile) + if not self._file_exists: + return expect(infile, "No postprocessing file defined in {}".format(files.filename)) schema = files.get_schema("POSTPROCESSING_SPEC_FILE") @@ -27,6 +30,8 @@ def __init__(self, infile=None, files=None): # Append the contents of $HOME/.cime/config_postprocessing.xml if it exists # This could cause problems if node matchs are repeated when only one is expected - infile = os.path.join(os.environ.get("HOME"), ".cime", "config_postprocessing.xml") + infile = os.path.join( + os.environ.get("HOME"), ".cime", "config_postprocessing.xml" + ) if os.path.exists(infile): - EntryID.read(self, infile) \ No newline at end of file + EntryID.read(self, infile) diff --git a/CIME/case/case.py b/CIME/case/case.py index 31959f54a68..dd93e15821d 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -1582,9 +1582,10 @@ def configure( workflow = Workflow(files=files) - env_postprocessing = self.get_env("postprocessing") postprocessing = Postprocessing(files=files) - env_postprocessing.add_elements_by_group(srcobj=postprocessing) + if postprocessing._file_exists: + env_postprocessing = self.get_env("postprocessing") + env_postprocessing.add_elements_by_group(srcobj=postprocessing) env_batch.set_batch_system(batch, batch_system_type=batch_system_type) From cac27fe47702c253b7d3fb0d46dc9354cfe085ac Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Mon, 3 Mar 2025 12:08:16 -0700 Subject: [PATCH 3/9] create env_postprocessing.xml if spec file exists If the spec file can not be found, remove env_postprocessing.xml from list of files that CIME will create --- CIME/case/case.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CIME/case/case.py b/CIME/case/case.py index dd93e15821d..d7f64a5f582 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -1586,6 +1586,13 @@ def configure( if postprocessing._file_exists: env_postprocessing = self.get_env("postprocessing") env_postprocessing.add_elements_by_group(srcobj=postprocessing) + else: + # if env_postprocessing does not exist, remove from self._files + self._files = [ + file + for file in self._files + if file.get_id() != "env_postprocessing.xml" + ] env_batch.set_batch_system(batch, batch_system_type=batch_system_type) From 3517d8a462afd4bc733f0f70b5f222f6e61ee03f Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Mon, 3 Mar 2025 12:21:28 -0700 Subject: [PATCH 4/9] Update action/cached to v3 Got a message that v2 was no longer supported, and the recommendation was to switch to v3 or v4. I chose v3 because that is also used in docs.yml --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1180a5fce9a..495bd88573e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -153,7 +153,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - name: Cache inputdata - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: /storage/inputdata key: inputdata-2 From d393f0c5b19d39d1eeb596a647b5612c1f04b119 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Tue, 4 Mar 2025 09:35:53 -0700 Subject: [PATCH 5/9] Move logic to remove empty env_postprocessing.xml This is now done in case.flush() since so many commands create this file (we don't just want to remove it after creating a new case because it comes back after running case.setup) --- CIME/case/case.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/CIME/case/case.py b/CIME/case/case.py index d7f64a5f582..361b05ef070 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -435,6 +435,14 @@ def flush(self, flushall=False): # do not flush if caseroot wasnt created return + if not os.path.isfile(self.get_value('POSTPROCESSING_SPEC_FILE')): + # Remove env_postprocessing.xml from self._files + self._files = [ + file + for file in self._files + if file.get_id() != "env_postprocessing.xml" + ] + for env_file in self._files: env_file.write(force_write=flushall) @@ -1586,13 +1594,6 @@ def configure( if postprocessing._file_exists: env_postprocessing = self.get_env("postprocessing") env_postprocessing.add_elements_by_group(srcobj=postprocessing) - else: - # if env_postprocessing does not exist, remove from self._files - self._files = [ - file - for file in self._files - if file.get_id() != "env_postprocessing.xml" - ] env_batch.set_batch_system(batch, batch_system_type=batch_system_type) From b2bbd83ed06f86bdf05a85eeb85d58e5a4cc546e Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Tue, 4 Mar 2025 09:49:08 -0700 Subject: [PATCH 6/9] Fix CI --- CIME/case/case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/case/case.py b/CIME/case/case.py index 361b05ef070..c26d2a3d29d 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -435,7 +435,7 @@ def flush(self, flushall=False): # do not flush if caseroot wasnt created return - if not os.path.isfile(self.get_value('POSTPROCESSING_SPEC_FILE')): + if not os.path.isfile(self.get_value("POSTPROCESSING_SPEC_FILE")): # Remove env_postprocessing.xml from self._files self._files = [ file From f9da5bb109984d1c3d16ea566e27c3596cb26843 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Tue, 4 Mar 2025 10:53:26 -0700 Subject: [PATCH 7/9] Only append EnvPostprocessing in two situations: In Case.read_xml(), we only want to include EnvPostprocessing object if one of the following is true: 1. The case is being created (added Case._existing_case to make it easy to check) 2. env_postprocessing.xml exists (which should only be the case if the file in POSTPROCESSING_SPEC_FILE exists) --- CIME/XML/postprocessing.py | 4 ++-- CIME/case/case.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CIME/XML/postprocessing.py b/CIME/XML/postprocessing.py index c52b499b5fd..b3bd6afdf28 100644 --- a/CIME/XML/postprocessing.py +++ b/CIME/XML/postprocessing.py @@ -19,8 +19,8 @@ def __init__(self, infile=None, files=None): files = Files() if infile is None: infile = files.get_value("POSTPROCESSING_SPEC_FILE") - self._file_exists = os.path.isfile(infile) - if not self._file_exists: + self.file_exists = os.path.isfile(infile) + if not self.file_exists: return expect(infile, "No postprocessing file defined in {}".format(files.filename)) diff --git a/CIME/case/case.py b/CIME/case/case.py index c26d2a3d29d..0905e31f906 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -111,6 +111,7 @@ def __init__(self, case_root=None, read_only=True, record=False, non_local=False case_root ), ) + self._existing_case = os.path.isdir(case_root) self._caseroot = case_root logger.debug("Initializing Case.") @@ -358,9 +359,10 @@ def read_xml(self): self._env_entryid_files.append( EnvWorkflow(self._caseroot, read_only=self._force_read_only) ) - self._env_entryid_files.append( - EnvPostprocessing(self._caseroot, read_only=self._force_read_only) - ) + if not self._existing_case or os.path.isfile("env_postprocessing.xml"): + self._env_entryid_files.append( + EnvPostprocessing(self._caseroot, read_only=self._force_read_only) + ) if os.path.isfile(os.path.join(self._caseroot, "env_test.xml")): self._env_entryid_files.append( @@ -1591,7 +1593,7 @@ def configure( workflow = Workflow(files=files) postprocessing = Postprocessing(files=files) - if postprocessing._file_exists: + if postprocessing.file_exists: env_postprocessing = self.get_env("postprocessing") env_postprocessing.add_elements_by_group(srcobj=postprocessing) From abbff8a48dc59d9234b3ad2beda517315806aeaf Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Tue, 4 Mar 2025 13:07:40 -0700 Subject: [PATCH 8/9] Update logic for removing env_postprocessing.xml pytest was raising flags in Case.flush() [maybe because of the way make_valid_case() created a new case directory?]; I'm sure there's a far more pythonic way to write this logic, but I need to avoid os.path.isfile(None) --- CIME/case/case.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CIME/case/case.py b/CIME/case/case.py index 0905e31f906..8ec0708198b 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -437,7 +437,12 @@ def flush(self, flushall=False): # do not flush if caseroot wasnt created return - if not os.path.isfile(self.get_value("POSTPROCESSING_SPEC_FILE")): + _postprocessing_spec_file = self.get_value("POSTPROCESSING_SPEC_FILE") + if _postprocessing_spec_file is not None: + have_postprocessing = os.path.isfile(_postprocessing_spec_file) + else: + have_postprocessing = False + if not have_postprocessing: # Remove env_postprocessing.xml from self._files self._files = [ file From b1f8a446b0bfa223c4759d073d723fcf65dc4d3e Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Tue, 4 Mar 2025 13:29:18 -0700 Subject: [PATCH 9/9] Model may not define POSTPROCESSING_SPEC_FILE In postprocessing.py, files.get_value("POSTPROCESSING_SPEC_FILE") will return None if the model does not have this variable defined. In that case, we don't want to call os.path.isfile(), we just want to set file_exists=False --- CIME/XML/postprocessing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CIME/XML/postprocessing.py b/CIME/XML/postprocessing.py index b3bd6afdf28..3287d145142 100644 --- a/CIME/XML/postprocessing.py +++ b/CIME/XML/postprocessing.py @@ -19,7 +19,10 @@ def __init__(self, infile=None, files=None): files = Files() if infile is None: infile = files.get_value("POSTPROCESSING_SPEC_FILE") - self.file_exists = os.path.isfile(infile) + if infile is not None: + self.file_exists = os.path.isfile(infile) + else: + self.file_exists = False if not self.file_exists: return expect(infile, "No postprocessing file defined in {}".format(files.filename))