ctsm5.3.069: Add SystemTests to run subset_data and then CTSM

.gitmodules

@@ -91,8 +91,8 @@ fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
 
 [submodule "cdeps"]
 path = components/cdeps
-url = https://github.com/ESCOMP/CDEPS.git
-fxtag = cdeps1.0.75
+url = https://github.com/samsrabin/CDEPS.git
+fxtag = 304d112e23019579013d08e57a463334ac8cac8f  # cx-length-filenames: Use CX-length filenames for all *_comp_nuopc.F90.
 fxrequired = ToplevelRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git

cime_config/SystemTests/subsetdata.py

@@ -0,0 +1,72 @@
+"""
+Parent class for CTSM-specific tests that first run the subset_data tool and then ensure
+that CTSM does not fail using the just-generated input files
+"""
+
+import os
+import sys
+import logging
+from CIME.SystemTests.system_tests_common import SystemTestsCommon
+from CIME.user_mod_support import apply_user_mods
+from CIME.XML.standard_module_setup import *
+
+# In case we need to import subset_data later
+_CTSM_PYTHON = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python"
+)
+sys.path.insert(1, _CTSM_PYTHON)
+
+logger = logging.getLogger(__name__)
+
+
+class SUBSETDATASHARED(SystemTestsCommon):
+    def __init__(self, case, subset_data_cmd):
+        """
+        initialize an object interface to the SMS system test
+        """
+        SystemTestsCommon.__init__(self, case)
+
+        # Check the test setup
+        if not self._case.get_value("LND_GRID") == "CLM_USRDAT":
+            raise RuntimeError("SUBSETDATA tests require resolution CLM_USRDAT")
+        if "serial" not in self._case.get_value("MPILIB"):
+            raise RuntimeError("SUBSETDATA tests require a serial MPILIB")
+        if "BGC-CROP" not in self._case.get_value("COMPSET"):
+            raise RuntimeError("SUBSETDATA tests require a BGC-CROP compset")
+
+        # Add standard subset_data arguments
+        out_dir = os.path.join(self._get_caseroot(), "subset_data_output")
+        self.usermods_dir = os.path.join(out_dir, "user_mods")
+        self.subset_data_cmd = subset_data_cmd + [
+            "--create-user-mods",
+            "--outdir",
+            out_dir,
+            "--user-mods-dir",
+            self.usermods_dir,
+            "--overwrite",
+        ]
+
+    def build_phase(self, sharedlib_only=False, model_only=False):
+
+        # Import subset_data. Do it here rather than at top because otherwise the import will
+        # be attempted even during RUN phase.
+        # pylint: disable=wrong-import-position,import-outside-toplevel
+        from ctsm.subset_data import main as subset_data
+
+        # Run the tool, if not already run, and apply usermods
+        if not os.path.exists(self.usermods_dir):
+            sys.argv = self.subset_data_cmd
+            subset_data()
+
+            # Required so that CTSM doesn't fail
+            user_nl_clm_path = os.path.join(self.usermods_dir, "user_nl_clm")
+            with open(user_nl_clm_path, "a", encoding="utf-8") as user_nl_clm:
+                user_nl_clm.write("\ncheck_dynpft_consistency = .false.\n")
+
+            # Apply the user mods
+            self._case.flush(flushall=True)
+            apply_user_mods(self._get_caseroot(), self.usermods_dir)
+            self._case.read_xml()
+
+        # Do the build
+        super().build_phase(sharedlib_only, model_only)

cime_config/SystemTests/subsetdatapoint.py

@@ -0,0 +1,38 @@
+"""
+CTSM-specific test that first runs the subset_data point tool and then ensures
+that CTSM does not fail using the just-generated input files
+"""
+
+from subsetdata import SUBSETDATASHARED
+
+
+class SUBSETDATAPOINT(SUBSETDATASHARED):
+    def __init__(self, case):
+        """
+        initialize an object interface to the SMS system test
+        """
+
+        lat = 45.402252
+        lon = -92.798085
+
+        # Don't need to include things that are added during SUBSETDATASHARED.__init__()
+        subset_data_cmd = [
+            "tools/site_and_regional/subset_data",
+            "point",
+            "--lat",
+            str(lat),
+            "--lon",
+            str(lon),
+            "--create-surface",
+            "--crop",
+            "--create-landuse",
+            "--surf-year",
+            "1850",
+            "--create-datm",
+            "--datm-syr",
+            "1901",
+            "--datm-eyr",
+            "1901",
+        ]
+
+        super().__init__(case, subset_data_cmd)

cime_config/SystemTests/subsetdataregion.py

@@ -0,0 +1,41 @@
+"""
+CTSM-specific test that first runs the subset_data region tool and then ensures
+that CTSM does not fail using the just-generated input files
+"""
+
+from subsetdata import SUBSETDATASHARED
+
+
+class SUBSETDATAREGION(SUBSETDATASHARED):
+    def __init__(self, case):
+        """
+        initialize an object interface to the SMS system test
+        """
+
+        lat1 = -9
+        lat2 = -7
+        lon1 = 291
+        lon2 = 293
+
+        # Don't need to include things that are added during SUBSETDATASHARED.__init__()
+        subset_data_cmd = [
+            "tools/site_and_regional/subset_data",
+            "region",
+            "--lat1",
+            str(lat1),
+            "--lat2",
+            str(lat2),
+            "--lon1",
+            str(lon1),
+            "--lon2",
+            str(lon2),
+            "--create-mesh",
+            "--create-domain",
+            "--create-surface",
+            "--crop",
+            "--create-landuse",
+            "--surf-year",
+            "1850",
+        ]
+
+        super().__init__(case, subset_data_cmd)

cime_config/config_tests.xml

@@ -35,6 +35,26 @@ This defines various CTSM-specific system tests
     <HIST_N>$STOP_N</HIST_N>
   </test>
 
+  <test NAME="SUBSETDATAPOINT">
+    <DESC>Run CTSM with files generated by the subset_data point tool</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
+  <test NAME="SUBSETDATAREGION">
+    <DESC>Run CTSM with files generated by the subset_data region tool</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>never</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+  </test>
+
   <test NAME="LGRAIN2">
     <DESC>CTSM Land model test to ensure that we can allocate and use a second grain pool</DESC>
     <INFO_DBUG>1</INFO_DBUG>

cime_config/testdefs/testlist_clm.xml

@@ -13,6 +13,7 @@
         rxcropmaturity: Short tests to be run during development related to prescribed crop calendars
         matrixcn: Tests exercising the matrix-CN capability
         aux_clm_mpi_serial: aux_clm tests using mpi-serial. Useful for redoing tests that failed due to https://github.com/ESCOMP/CTSM/issues/2916, after having replaced libraries/mpi-serial with a fresh copy.
+        subset_data: Tests exercising the subset_data tool and running CTSM with its output
 -->
 <testlist version="2.0">
   <test name="ERI_D_Ld9" grid="f10_f10_mg37" compset="I1850Clm60Bgc" testmods="clm/default">
@@ -4129,6 +4130,34 @@
     </options>
   </test>
 
+  <test name="SUBSETDATAPOINT_Ld5_D_Mmpi-serial" grid="CLM_USRDAT" compset="I2000Clm60BgcCropCrujra" testmods="clm-default">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+      <machine name="derecho" compiler="intel" category="aux_clm_mpi_serial"/>
+      <machine name="derecho" compiler="intel" category="clm_pymods"/>
+      <machine name="derecho" compiler="intel" category="subset_data"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:20:00</option>
+      <option name="comment">Smoke test that first runs the subset_data point tool and then ensures that CTSM does not fail using the just-generated user mods.</option>
+      <option name="comment">This test invokes python code, so it should be run whenever changing python code (in addition to being run as part of aux_clm).</option>
+    </options>
+  </test>
+
+  <test name="SUBSETDATAREGION_Ld5_D_Mmpi-serial" grid="CLM_USRDAT" compset="I2000Clm60BgcCropCrujra" testmods="clm-default">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_clm"/>
+      <machine name="derecho" compiler="intel" category="aux_clm_mpi_serial"/>
+      <machine name="derecho" compiler="intel" category="clm_pymods"/>
+      <machine name="derecho" compiler="intel" category="subset_data"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:20:00</option>
+      <option name="comment">Smoke test that first runs the subset_data regiontool and then ensures that CTSM does not fail using the just-generated user mods.</option>
+      <option name="comment">This test invokes python code, so it should be run whenever changing python code (in addition to being run as part of aux_clm).</option>
+    </options>
+  </test>
+
   <test name="LILACSMOKE_D_Ld2" grid="f10_f10_mg37" compset="I2000Ctsm50NwpSpAsRs" testmods="clm-lilac">
     <machines>
       <machine name="derecho" compiler="intel" category="aux_clm"/>

python/ctsm/run_sys_tests.py

@@ -749,6 +749,14 @@ def _check_py_env(test_attributes):
         except ModuleNotFoundError as err:
             raise ModuleNotFoundError("modify_fsurdat" + err_msg) from err
 
+    # Check requirements for using subset_data Python module, if needed
+    subset_data_users = ["SUBSETDATAPOINT", "SUBSETDATAREGION"]
+    if any(any(u in t for u in subset_data_users) for t in test_attributes):
+        try:
+            import ctsm.subset_data
+        except ModuleNotFoundError as err:
+            raise ModuleNotFoundError("subset_data" + err_msg) from err
+
     # Check requirements for RXCROPMATURITY, if needed
     if any("RXCROPMATURITY" in t for t in test_attributes):
         try:

src/dyn_subgrid/dynFileMod.F90

@@ -7,6 +7,7 @@ module dynFileMod
   !
   ! !USES:
   use shr_log_mod    , only : errMsg => shr_log_errMsg
+  use clm_varctl     , only : fname_len
   use dynTimeInfoMod , only : time_info_type, year_position_type
   use ncdio_pio      , only : file_desc_t, ncd_pio_openfile, ncd_inqdid, ncd_inqdlen, ncd_io
   use abortutils     , only : endrun
@@ -56,7 +57,7 @@ type(dyn_file_type) function constructor(filename, year_position)
     type(year_position_type) , intent(in) :: year_position
     !
     ! !LOCAL VARIABLES:
-    character(len=256) :: locfn      ! local file name
+    character(len=fname_len) :: locfn  ! local file name
     integer :: ier                   ! error code
     integer :: ntimes                ! number of time samples
     integer :: varid                 ! netcdf variable ID

src/main/surfrdMod.F90

@@ -15,6 +15,7 @@ module surfrdMod
   use clm_varcon      , only : grlnd
   use clm_varctl      , only : iulog
   use clm_varctl      , only : use_cndv, use_crop, use_fates
+  use clm_varctl      , only : fname_len
   use surfrdUtilsMod  , only : check_sums_equal_1, apply_convert_ocean_to_land, collapse_crop_types
   use surfrdUtilsMod  , only : collapse_to_dominant, collapse_crop_var, collapse_individual_lunits
   use ncdio_pio       , only : file_desc_t, var_desc_t, ncd_pio_openfile, ncd_pio_closefile
@@ -233,7 +234,7 @@ subroutine surfrd_get_data (begg, endg, ldomain, lfsurdat, lhillslope_file, actu
     character(len=*), intent(in) :: lhillslope_file ! hillslope dataset filename
     !
     ! !LOCAL VARIABLES:
-    character(len=256):: locfn                ! local file name
+    character(len=fname_len)  :: locfn        ! local file name
     integer, parameter :: n_dom_urban = 1     ! # of dominant urban landunits
     type(file_desc_t) :: ncid                 ! netcdf id for lfsurdat
     type(file_desc_t) :: ncid_hillslope       ! netcdf id for lhillslope_file
@@ -359,7 +360,7 @@ subroutine surfrd_get_num_patches (lfsurdat, actual_maxsoil_patches, actual_nump
 
     !
     ! !LOCAL VARIABLES:
-    character(len=256):: locfn                ! local file name
+    character(len=fname_len) :: locfn         ! local file name
     type(file_desc_t) :: ncid                 ! netcdf file id
     integer :: dimid                          ! netCDF dimension id
     logical :: cft_dim_exists                 ! dimension exists on dataset
@@ -437,7 +438,7 @@ subroutine surfrd_get_nlevurb (lfsurdat, actual_nlevurb)
     integer, intent(out) :: actual_nlevurb    ! nlevurb from surface dataset
     !
     ! !LOCAL VARIABLES:
-    character(len=256):: locfn                ! local file name
+    character(len=fname_len)  :: locfn        ! local file name
     type(file_desc_t) :: ncid                 ! netcdf file id
     integer :: dimid                          ! netCDF dimension id
     character(len=32) :: subname = 'surfrd_get_nlevurb'  ! subroutine name
@@ -1163,7 +1164,6 @@ subroutine surfrd_lakemask(begg, endg)
     ! !USES:
      use clm_instur           , only : pct_lake_max
      use dynSubgridControlMod , only : get_flanduse_timeseries
-     use clm_varctl           , only : fname_len
      use fileutils            , only : getfil
     !
     ! !ARGUMENTS:
@@ -1172,7 +1172,7 @@ subroutine surfrd_lakemask(begg, endg)
     !
     ! !LOCAL VARIABLES:
     type(file_desc_t)         :: ncid_dynuse          ! netcdf id for landuse timeseries file
-    character(len=256)        :: locfn                ! local file name
+    character(len=fname_len)  :: locfn                ! local file name
     character(len=fname_len)  :: fdynuse              ! landuse.timeseries filename
     logical                   :: readvar
     !
@@ -1218,7 +1218,6 @@ subroutine surfrd_urbanmask(begg, endg)
     ! !USES:
      use clm_instur           , only : pct_urban_max
      use dynSubgridControlMod , only : get_flanduse_timeseries
-     use clm_varctl           , only : fname_len
      use fileutils            , only : getfil
     !
     ! !ARGUMENTS:
@@ -1227,7 +1226,7 @@ subroutine surfrd_urbanmask(begg, endg)
     !
     ! !LOCAL VARIABLES:
     type(file_desc_t)         :: ncid_dynuse          ! netcdf id for landuse timeseries file
-    character(len=256)        :: locfn                ! local file name
+    character(len=fname_len)  :: locfn                ! local file name
     character(len=fname_len)  :: fdynuse              ! landuse.timeseries filename
     logical                   :: readvar
     !