diff --git a/.gitmodules b/.gitmodules index 85ba78a173..ba76bf816a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -76,7 +76,7 @@ fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git [submodule "cime"] path = cime url = https://github.com/ESMCI/cime -fxtag = cime6.1.111 +fxtag = cime6.1.112 fxrequired = ToplevelRequired # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed fxDONOTUSEurl = https://github.com/ESMCI/cime diff --git a/cime b/cime index bbd4addfc4..6fb4e9e1a3 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit bbd4addfc4fb4b1031f71a2111a2baae424f98fd +Subproject commit 6fb4e9e1a3bcb72f9251fc86b90660fc59f7da12 diff --git a/cime_config/SystemTests/subsetdata.py b/cime_config/SystemTests/subsetdata.py new file mode 100644 index 0000000000..6861030b04 --- /dev/null +++ b/cime_config/SystemTests/subsetdata.py @@ -0,0 +1,100 @@ +""" +Parent class for CTSM-specific tests that first run the subset_data tool and then ensure +that CTSM does not fail using the just-generated input files +""" + +import os +import sys +import logging +from CIME.SystemTests.system_tests_common import SystemTestsCommon +from CIME.user_mod_support import apply_user_mods +from CIME.XML.standard_module_setup import * + +# In case we need to import subset_data later +_CTSM_PYTHON = os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python" +) +sys.path.insert(1, _CTSM_PYTHON) + +logger = logging.getLogger(__name__) + + +class SUBSETDATASHARED(SystemTestsCommon): + def __init__(self, case, subset_data_cmd): + """ + initialize an object interface to the SMS system test + """ + SystemTestsCommon.__init__(self, case) + + # Check the test setup + if not self._case.get_value("LND_GRID") == "CLM_USRDAT": + raise RuntimeError("SUBSETDATA tests require resolution CLM_USRDAT") + if "serial" not in self._case.get_value("MPILIB"): + raise RuntimeError("SUBSETDATA tests require a serial MPILIB") + if "BGC-CROP" not in self._case.get_value("COMPSET"): + raise RuntimeError("SUBSETDATA tests require a BGC-CROP compset") + + # Add standard subset_data arguments + out_dir = os.path.join(self._get_caseroot(), "subset_data_output") + self.usermods_dir = os.path.join(out_dir, "user_mods") + self.subset_data_cmd = subset_data_cmd + [ + "--create-user-mods", + "--outdir", + out_dir, + "--user-mods-dir", + self.usermods_dir, + "--overwrite", + ] + + # Run subset_data, if needed. + # It's needed during SETUP and/or NLCOMP phases if comparing/generating a baseline because + # the namelist comparison will require generating a namelist, and that will fail if we + # haven't specified our custom fsurdat and other stuff. By calling self._run_subset_data() + # only if the usermods directory doesn't yet exist, we avoid it being called every time the + # test class is initialized (which happens, e.g., in RUN phase). + # if not os.path.exists(self.usermods_dir): + # self._run_subset_data() + + def build_phase(self, sharedlib_only=False, model_only=False): + """ + Run subset_data and then build the model + """ + + # Run subset_data. + # build_phase gets called twice: + # - once with sharedlib_only = True and + # - once with model_only = True + # Because we only need subset_data run once, we only do it for the sharedlib_only call. + # We could also check for the existence of the subset_data outputs, but that might lead to + # a situation where the user expects subset_data to be called but it's not. Better to run + # unnecessarily (e.g., if you fixed some FORTRAN code and just need to rebuild). + # In that same vein, yes we did run subset_data during the first time this test case was + # initialized (see __init__() above), but we're doing it again here just to be safe. + if sharedlib_only: + self._run_subset_data() + + # Do the build + self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) + + def _run_subset_data(self): + """ + Run subset_data + """ + # Import subset_data. Do it here rather than at top because otherwise the import will + # be attempted even during RUN phase. + # pylint: disable=wrong-import-position,import-outside-toplevel + from ctsm.subset_data import main as subset_data + + # Run subset_data + sys.argv = self.subset_data_cmd + subset_data() + + # Required so that CTSM doesn't fail + user_nl_clm_path = os.path.join(self.usermods_dir, "user_nl_clm") + with open(user_nl_clm_path, "a", encoding="utf-8") as user_nl_clm: + user_nl_clm.write("\ncheck_dynpft_consistency = .false.\n") + + # Apply the user mods + self._case.flush(flushall=True) + apply_user_mods(self._get_caseroot(), self.usermods_dir) + self._case.read_xml() diff --git a/cime_config/SystemTests/subsetdatapoint.py b/cime_config/SystemTests/subsetdatapoint.py new file mode 100644 index 0000000000..6b735809cd --- /dev/null +++ b/cime_config/SystemTests/subsetdatapoint.py @@ -0,0 +1,38 @@ +""" +CTSM-specific test that first runs the subset_data point tool and then ensures +that CTSM does not fail using the just-generated input files +""" + +from subsetdata import SUBSETDATASHARED + + +class SUBSETDATAPOINT(SUBSETDATASHARED): + def __init__(self, case): + """ + initialize an object interface to the SMS system test + """ + + lat = 45.402252 + lon = -92.798085 + + # Don't need to include things that are added during SUBSETDATASHARED.__init__() + subset_data_cmd = [ + "tools/site_and_regional/subset_data", + "point", + "--lat", + str(lat), + "--lon", + str(lon), + "--create-surface", + "--crop", + "--create-landuse", + "--surf-year", + "1850", + "--create-datm", + "--datm-syr", + "1901", + "--datm-eyr", + "1901", + ] + + super().__init__(case, subset_data_cmd) diff --git a/cime_config/SystemTests/subsetdataregion.py b/cime_config/SystemTests/subsetdataregion.py new file mode 100644 index 0000000000..e0c8144b3a --- /dev/null +++ b/cime_config/SystemTests/subsetdataregion.py @@ -0,0 +1,41 @@ +""" +CTSM-specific test that first runs the subset_data region tool and then ensures +that CTSM does not fail using the just-generated input files +""" + +from subsetdata import SUBSETDATASHARED + + +class SUBSETDATAREGION(SUBSETDATASHARED): + def __init__(self, case): + """ + initialize an object interface to the SMS system test + """ + + lat1 = -9 + lat2 = -7 + lon1 = 291 + lon2 = 293 + + # Don't need to include things that are added during SUBSETDATASHARED.__init__() + subset_data_cmd = [ + "tools/site_and_regional/subset_data", + "region", + "--lat1", + str(lat1), + "--lat2", + str(lat2), + "--lon1", + str(lon1), + "--lon2", + str(lon2), + "--create-mesh", + "--create-domain", + "--create-surface", + "--crop", + "--create-landuse", + "--surf-year", + "1850", + ] + + super().__init__(case, subset_data_cmd) diff --git a/cime_config/config_tests.xml b/cime_config/config_tests.xml index 12859b9131..2d7b1fb2bb 100644 --- a/cime_config/config_tests.xml +++ b/cime_config/config_tests.xml @@ -35,6 +35,26 @@ This defines various CTSM-specific system tests $STOP_N + + Run CTSM with files generated by the subset_data point tool + 1 + FALSE + FALSE + never + $STOP_OPTION + $STOP_N + + + + Run CTSM with files generated by the subset_data region tool + 1 + FALSE + FALSE + never + $STOP_OPTION + $STOP_N + + CTSM Land model test to ensure that we can allocate and use a second grain pool 1 diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 595015c2db..be4b1c00c6 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -13,6 +13,7 @@ rxcropmaturity: Short tests to be run during development related to prescribed crop calendars matrixcn: Tests exercising the matrix-CN capability aux_clm_mpi_serial: aux_clm tests using mpi-serial. Useful for redoing tests that failed due to https://github.com/ESCOMP/CTSM/issues/2916, after having replaced libraries/mpi-serial with a fresh copy. + subset_data: Tests exercising the subset_data tool and running CTSM with its output decomp_init: Initialization tests specifically for examining the PE layout decomposition initialization interim_restart: Tests having to do with interim restart capability. --> @@ -4141,6 +4142,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/ChangeLog b/doc/ChangeLog index 52d853c436..df709ebd94 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,77 @@ =============================================================== +Tag name: ctsm5.3.069 +Originator(s): samrabin (Sam Rabin, UCAR/TSS) +Date: Tue Aug 12 09:52:59 MDT 2025 +One-line Summary: Add SystemTests to run subset_data and then CTSM + +Purpose and description of changes +---------------------------------- + +Adds SUBSETDATAPOINT and SUBSETDATAREGION tests to aux_clm (and new subset_data suite). These run subset_data for either a point or a six-cell region, then run CTSM with the outputs. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + +[ ] clm6_0 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed +---------- + +List of CTSM issues fixed (include CTSM Issue # and description): +- Resolves [Issue #1491: Add two new tests that run subset_data and then run a case from it](https://github.com/ESCOMP/CTSM/issues/1491) + + +Notes of particular relevance for developers: +--------------------------------------------- +NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + +Changes to tests or testing: +- Adds test SUBSETDATAPOINT_Ld5_D_Mmpi-serial.CLM_USRDAT.I2000Clm60BgcCropCrujra.derecho_intel.clm-default +- Adds test SUBSETDATAREGION_Ld5_D_Mmpi-serial.CLM_USRDAT.I2000Clm60BgcCropCrujra.derecho_intel.clm-default +- Adds new subset_data suite with those tests in it +- Those tests are also in aux_clm + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + derecho - + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + derecho - Unit, black, and pylint checks pass. Two system tests fail due to recent conda and mamba updates on Derecho; these have been fixed on b4b-dev with [Pull Request #3403: Fix py_env_create and tests by samsrabin](https://github.com/ESCOMP/CTSM/pull/3403) + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + derecho ----- OK + izumi ------- OK + + +Other details +------------- + +Pull Requests that document the changes (include PR ids): +- [Pull Request #3292: ctsm5.3.069: Add SystemTests to run subset_data and then CTSM by samsrabin](https://github.com/ESCOMP/CTSM/pull/3292) + +=============================================================== +=============================================================== Tag name: ctsm5.3.068 Originator(s): slevis (Samuel Levis,UCAR/TSS,303-665-1310) Date: Mon 11 Aug 2025 02:27:39 PM MDT diff --git a/doc/ChangeSum b/doc/ChangeSum index 442484253a..33389e00c5 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.3.069 samrabin 08/12/2025 Add SystemTests to run subset_data and then CTSM ctsm5.3.068 slevis 08/11/2025 Change megan_use_gamma_sm to default false ctsm5.3.067 samrabin 08/08/2025 (Mostly) fix interim restarts ctsm5.3.066 slevis 08/08/2025 Merge b4b-dev to master diff --git a/python/ctsm/run_sys_tests.py b/python/ctsm/run_sys_tests.py index 8108668246..e19bcd9b98 100644 --- a/python/ctsm/run_sys_tests.py +++ b/python/ctsm/run_sys_tests.py @@ -749,6 +749,14 @@ def _check_py_env(test_attributes): except ModuleNotFoundError as err: raise ModuleNotFoundError("modify_fsurdat" + err_msg) from err + # Check requirements for using subset_data Python module, if needed + subset_data_users = ["SUBSETDATAPOINT", "SUBSETDATAREGION"] + if any(any(u in t for u in subset_data_users) for t in test_attributes): + try: + import ctsm.subset_data + except ModuleNotFoundError as err: + raise ModuleNotFoundError("subset_data" + err_msg) from err + # Check requirements for RXCROPMATURITY, if needed if any("RXCROPMATURITY" in t for t in test_attributes): try: diff --git a/src/dyn_subgrid/dynFileMod.F90 b/src/dyn_subgrid/dynFileMod.F90 index d7f22f9dcd..ecb2d111d9 100644 --- a/src/dyn_subgrid/dynFileMod.F90 +++ b/src/dyn_subgrid/dynFileMod.F90 @@ -7,6 +7,7 @@ module dynFileMod ! ! !USES: use shr_log_mod , only : errMsg => shr_log_errMsg + use clm_varctl , only : fname_len use dynTimeInfoMod , only : time_info_type, year_position_type use ncdio_pio , only : file_desc_t, ncd_pio_openfile, ncd_inqdid, ncd_inqdlen, ncd_io use abortutils , only : endrun @@ -56,7 +57,7 @@ type(dyn_file_type) function constructor(filename, year_position) type(year_position_type) , intent(in) :: year_position ! ! !LOCAL VARIABLES: - character(len=256) :: locfn ! local file name + character(len=fname_len) :: locfn ! local file name integer :: ier ! error code integer :: ntimes ! number of time samples integer :: varid ! netcdf variable ID diff --git a/src/main/surfrdMod.F90 b/src/main/surfrdMod.F90 index c70ec28fa0..188773dfcd 100644 --- a/src/main/surfrdMod.F90 +++ b/src/main/surfrdMod.F90 @@ -15,6 +15,7 @@ module surfrdMod use clm_varcon , only : grlnd use clm_varctl , only : iulog use clm_varctl , only : use_cndv, use_crop, use_fates + use clm_varctl , only : fname_len use surfrdUtilsMod , only : check_sums_equal_1, apply_convert_ocean_to_land, collapse_crop_types use surfrdUtilsMod , only : collapse_to_dominant, collapse_crop_var, collapse_individual_lunits use ncdio_pio , only : file_desc_t, var_desc_t, ncd_pio_openfile, ncd_pio_closefile @@ -233,7 +234,7 @@ subroutine surfrd_get_data (begg, endg, ldomain, lfsurdat, lhillslope_file, actu character(len=*), intent(in) :: lhillslope_file ! hillslope dataset filename ! ! !LOCAL VARIABLES: - character(len=256):: locfn ! local file name + character(len=fname_len) :: locfn ! local file name integer, parameter :: n_dom_urban = 1 ! # of dominant urban landunits type(file_desc_t) :: ncid ! netcdf id for lfsurdat type(file_desc_t) :: ncid_hillslope ! netcdf id for lhillslope_file @@ -359,7 +360,7 @@ subroutine surfrd_get_num_patches (lfsurdat, actual_maxsoil_patches, actual_nump ! ! !LOCAL VARIABLES: - character(len=256):: locfn ! local file name + character(len=fname_len) :: locfn ! local file name type(file_desc_t) :: ncid ! netcdf file id integer :: dimid ! netCDF dimension id logical :: cft_dim_exists ! dimension exists on dataset @@ -437,7 +438,7 @@ subroutine surfrd_get_nlevurb (lfsurdat, actual_nlevurb) integer, intent(out) :: actual_nlevurb ! nlevurb from surface dataset ! ! !LOCAL VARIABLES: - character(len=256):: locfn ! local file name + character(len=fname_len) :: locfn ! local file name type(file_desc_t) :: ncid ! netcdf file id integer :: dimid ! netCDF dimension id character(len=32) :: subname = 'surfrd_get_nlevurb' ! subroutine name @@ -1163,7 +1164,6 @@ subroutine surfrd_lakemask(begg, endg) ! !USES: use clm_instur , only : pct_lake_max use dynSubgridControlMod , only : get_flanduse_timeseries - use clm_varctl , only : fname_len use fileutils , only : getfil ! ! !ARGUMENTS: @@ -1172,7 +1172,7 @@ subroutine surfrd_lakemask(begg, endg) ! ! !LOCAL VARIABLES: type(file_desc_t) :: ncid_dynuse ! netcdf id for landuse timeseries file - character(len=256) :: locfn ! local file name + character(len=fname_len) :: locfn ! local file name character(len=fname_len) :: fdynuse ! landuse.timeseries filename logical :: readvar ! @@ -1218,7 +1218,6 @@ subroutine surfrd_urbanmask(begg, endg) ! !USES: use clm_instur , only : pct_urban_max use dynSubgridControlMod , only : get_flanduse_timeseries - use clm_varctl , only : fname_len use fileutils , only : getfil ! ! !ARGUMENTS: @@ -1227,7 +1226,7 @@ subroutine surfrd_urbanmask(begg, endg) ! ! !LOCAL VARIABLES: type(file_desc_t) :: ncid_dynuse ! netcdf id for landuse timeseries file - character(len=256) :: locfn ! local file name + character(len=fname_len) :: locfn ! local file name character(len=fname_len) :: fdynuse ! landuse.timeseries filename logical :: readvar !