b4b-dev: Plumber2 Implementation

.gitignore

@@ -92,8 +92,18 @@ unit_test_build
 /tools/site_and_regional/listing.csv
 /tools/site_and_regional/????/
 /tools/site_and_regional/????.ad/
+/tools/site_and_regional/????.*.ad/
 /tools/site_and_regional/????.postad/
+/tools/site_and_regional/????.*.postad/
 /tools/site_and_regional/????.transient/
+/tools/site_and_regional/????.*.transient/
+/tools/site_and_regional/??-???/
+/tools/site_and_regional/??-???.ad/
+/tools/site_and_regional/??-???.*.ad/
+/tools/site_and_regional/??-???.postad/
+/tools/site_and_regional/??-???.*.postad/
+/tools/site_and_regional/??-???.transient/
+/tools/site_and_regional/??-???.*.transient/
 /tools/site_and_regional/archive/
 
 # build output

python/ctsm/site_and_regional/neon_site.py

@@ -1,5 +1,5 @@
 """
-This module contains the NeonSite class and class functions which are used in run_neon.py
+This module contains the NeonSite class and class functions which are used in run_tower.py
 """
 
 # Import libraries
@@ -57,7 +57,6 @@ def run_case(
         base_case_root,
         run_type,
         prism,
-        run_length,
         user_version,
         tower_type=None,
         user_mods_dirs=None,
@@ -78,8 +77,6 @@ def run_case(
             transient, post_ad, or ad case, default transient
         prism: bool, opt
             if True, use PRISM precipitation, default False
-        run_length: str, opt
-            length of run, default '4Y'
         user_version: str, opt
             default 'latest'
         overwrite: bool, opt
@@ -102,7 +99,6 @@ def run_case(
             base_case_root,
             run_type,
             prism,
-            run_length,
             user_version,
             tower_type,
             user_mods_dirs,

python/ctsm/site_and_regional/plumber_site.py

@@ -0,0 +1,115 @@
+"""
+This module contains the Plumber2Site class and class functions which are used in run_tower.py
+"""
+
+# Import libraries
+import logging
+import os
+import sys
+
+# Get the ctsm util tools and then the cime tools.
+_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python"))
+sys.path.insert(1, _CTSM_PYTHON)
+
+# -- import local classes for this script
+# pylint: disable=wrong-import-position
+from ctsm.site_and_regional.tower_site import TowerSite
+
+# pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order
+from ctsm import add_cime_to_path
+from ctsm.path_utils import path_to_ctsm_root
+
+from CIME import build
+from CIME.case import Case
+from CIME.utils import safe_copy, expect, symlink_force
+
+logger = logging.getLogger(__name__)
+
+
+# pylint: disable=too-many-instance-attributes
+class Plumber2Site(TowerSite):
+    """
+    A class for encapsulating plumber sites.
+    """
+
+    def build_base_case(
+        self,
+        cesmroot,
+        output_root,
+        res,
+        compset,
+        user_mods_dirs=None,
+        overwrite=False,
+        setup_only=False,
+    ):
+        if user_mods_dirs is None:
+            user_mods_dirs = [
+                os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "PLUMBER2", self.name)
+            ]
+        case_path = super().build_base_case(cesmroot, output_root, res, compset, user_mods_dirs)
+
+        return case_path
+
+    # pylint: disable=too-many-statements
+    def run_case(
+        self,
+        base_case_root,
+        run_type,
+        prism,
+        user_version,
+        tower_type=None,
+        user_mods_dirs=None,
+        overwrite=False,
+        setup_only=False,
+        no_batch=False,
+        rerun=False,
+        experiment=False,
+    ):
+        """
+        Run case.
+
+        Args:
+        self
+        base_case_root: str, opt
+            file path of base case
+        run_type: str, opt
+            transient, post_ad, or ad case, default ad
+            (ad case is default because PLUMBER requires spinup)
+        prism: bool, opt
+            if True, use PRISM precipitation, default False
+            Note: only supported for NEON sites
+        user_version: str, opt
+            default 'latest'; this could be useful later
+            This is currently only implemented with neon (not plumber) sites
+        overwrite: bool, opt
+            default False
+        setup_only: bool, opt
+            default False; if True, set up but do not run case
+        no_batch: bool, opt
+            default False
+        rerun: bool, opt
+            default False
+        experiment: str, opt
+            name of experiment, default False
+        """
+        user_mods_dirs = [
+            os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "PLUMBER2", self.name)
+        ]
+        tower_type = "PLUMBER"
+        super().run_case(
+            base_case_root,
+            run_type,
+            prism,
+            user_version,
+            tower_type,
+            user_mods_dirs,
+            overwrite,
+            setup_only,
+            no_batch,
+            rerun,
+            experiment,
+        )
+
+    def set_ref_case(self, case):
+        super().set_ref_case(case)
+        return True  ### Check if super returns false, if this will still return True?

python/ctsm/site_and_regional/run_neon.py → python/ctsm/site_and_regional/run_tower.py

@@ -5,18 +5,18 @@
 |---------------------  Instructions  -----------------------------|
 |------------------------------------------------------------------|
 This is a wrapper script for running CTSM simulation for one or more
-neon sites.
+tower (neon or plumber) sites.
 
-This script is only for neon site and we will develop a more general
-code later.
+This script is only for supported tower sites and we will develop a
+more general code later.
 
 This script first creates and builds a generic base case.
-Next, it will clone the base_case for different neon sites and run
+Next, it will clone the base_case for different tower sites and run
 types to reduce the need to build ctsm everytime.
 
 This script will do the following:
     1) Create a generic base case for cloning.
-    2) Make the case for the specific neon site(s).
+    2) Make the case for the specific neon or plumber site(s).
     3) Make changes to the case, for:
         a. AD spinup
 	b. post-AD spinup
@@ -33,11 +33,10 @@
 
 -------------------------------------------------------------------
 To see the available options:
-    ./run_neon.py --help
+    ./run_tower.py --help
 -------------------------------------------------------------------
 """
 # TODO (NS)
-# - [ ]
 # - [ ] Case dependency and the ability to check case status
 # - [ ] If Case dependency works we don't need finidat given explicilty for post-ad and transient.
 
@@ -64,8 +63,9 @@
 # pylint: disable=wrong-import-position
 from ctsm.path_utils import path_to_ctsm_root
 from ctsm.download_utils import download_file
-from ctsm.site_and_regional.neon_arg_parse import get_parser
+from ctsm.site_and_regional.tower_arg_parse import get_parser
 from ctsm.site_and_regional.neon_site import NeonSite
+from ctsm.site_and_regional.plumber_site import Plumber2Site
 
 # pylint: disable=import-error, wildcard-import, wrong-import-order
 from standard_script_setup import *
@@ -130,7 +130,7 @@ def parse_neon_listing(listing_file, valid_neon_sites):
             # versions = tmp_df[7].unique()
             # print ("all versions available for ", site_name,":", *versions)
             latest_version = tmp_df[7].iloc[-1]
-            # print ("latests version available for ", site_name,":", latest_version)
+            # print ("latest version available for ", site_name,":", latest_version)
 
             tmp_df = tmp_df[tmp_df[7].str.contains(latest_version)]
             # -- remove .nc from the file names
@@ -167,9 +167,41 @@ def parse_neon_listing(listing_file, valid_neon_sites):
     return available_list
 
 
+def setup_plumber_data(valid_plumber_sites):
+    """
+    A function to set up plumber site objects
+    with dummy start and end years and months.
+    This allows us to use the list of plumber site objects.
+
+    Returns:
+        available_list :
+            list of plumber_site objects that is found
+    """
+
+    available_list = []
+
+    for site_name in valid_plumber_sites:
+
+        # start_year and end_year are set in shell commands, so these get overwritten
+        start_year = 8888
+        end_year = 9999
+        start_month = 1
+        end_month = 12
+
+        logger.debug("Valid plumber site %s found!", site_name)
+        finidat = None
+
+        plumber_site = Plumber2Site(
+            site_name, start_year, end_year, start_month, end_month, finidat
+        )
+        available_list.append(plumber_site)
+
+    return available_list
+
+
 def main(description):
     """
-    Determine valid neon sites. Make an output directory if it does not exist.
+    Determine valid tower sites. Make an output directory if it does not exist.
     Loop through requested sites and run CTSM at that site.
     """
     cesmroot = path_to_ctsm_root()
@@ -181,21 +213,28 @@ def main(description):
     )
     valid_neon_sites = sorted([v.split("/")[-1] for v in valid_neon_sites])
 
+    # Get the list of supported plumber sites from usermods
+    valid_plumber_sites = glob.glob(
+        os.path.join(cesmroot, "cime_config", "usermods_dirs", "PLUMBER2", "[!d]*")
+    )
+
+    valid_plumber_sites = sorted([v.split("/")[-1] for v in valid_plumber_sites])
+
     (
-        site_list,
+        neon_site_list,
+        plumber_site_list,
         output_root,
         run_type,
         experiment,
         prism,
         overwrite,
-        run_length,
         base_case_root,
         run_from_postad,
         setup_only,
         no_batch,
         rerun,
         user_version,
-    ) = get_parser(sys.argv, description, valid_neon_sites)
+    ) = get_parser(sys.argv, description, valid_neon_sites, valid_plumber_sites)
 
     if output_root:
         logger.debug("output_root : %s", output_root)
@@ -206,7 +245,7 @@ def main(description):
     available_list = check_neon_listing(valid_neon_sites)
 
     # =================================
-    # -- all neon sites can be cloned from one generic case
+    # -- all tower sites can be cloned from one generic case
     # -- so no need to define a base_case for every site.
 
     res = "CLM_USRDAT"
@@ -216,28 +255,55 @@ def main(description):
         compset = "I1PtClm60Bgc"
 
     # --  Looping over neon sites
+    if neon_site_list:
+        for neon_site in available_list:
+            if neon_site.name in neon_site_list:
+                if run_from_postad:
+                    neon_site.finidat = None
+                if not base_case_root:
+                    user_mods_dirs = None
+                    base_case_root = neon_site.build_base_case(
+                        cesmroot, output_root, res, compset, user_mods_dirs, overwrite, setup_only
+                    )
+                logger.info("-----------------------------------")
+                logger.info("Running CTSM for neon site : %s", neon_site.name)
+
+                neon_site.run_case(
+                    base_case_root,
+                    run_type,
+                    prism,
+                    user_version,
+                    overwrite=overwrite,
+                    setup_only=setup_only,
+                    no_batch=no_batch,
+                    rerun=rerun,
+                    experiment=experiment,
+                )
 
-    for neon_site in available_list:
-        if neon_site.name in site_list:
-            if run_from_postad:
-                neon_site.finidat = None
-            if not base_case_root:
-                user_mods_dirs = None
-                base_case_root = neon_site.build_base_case(
-                    cesmroot, output_root, res, compset, user_mods_dirs, overwrite, setup_only
+    # -- check for available plumber data:
+    available_plumber_list = setup_plumber_data(valid_plumber_sites)
+
+    # --  Looping over plumber sites
+    if plumber_site_list:
+        for plumber_site in available_plumber_list:
+            if plumber_site.name in plumber_site_list:
+                if run_from_postad:
+                    plumber_site.finidat = None
+                if not base_case_root:
+                    user_mods_dirs = None
+                    base_case_root = plumber_site.build_base_case(
+                        cesmroot, output_root, res, compset, user_mods_dirs, overwrite, setup_only
+                    )
+                logger.info("-----------------------------------")
+                logger.info("Running CTSM for plumber site : %s", plumber_site.name)
+                plumber_site.run_case(
+                    base_case_root,
+                    run_type,
+                    prism,
+                    user_version,
+                    overwrite=overwrite,
+                    setup_only=setup_only,
+                    no_batch=no_batch,
+                    rerun=rerun,
+                    experiment=experiment,
                 )
-            logger.info("-----------------------------------")
-            logger.info("Running CTSM for neon site : %s", neon_site.name)
-
-            neon_site.run_case(
-                base_case_root,
-                run_type,
-                prism,
-                run_length,
-                user_version,
-                overwrite=overwrite,
-                setup_only=setup_only,
-                no_batch=no_batch,
-                rerun=rerun,
-                experiment=experiment,
-            )