[Replaced] Logging and fixes for h2i GDD generation

cime_config/SystemTests/rxcropmaturity.py

@@ -86,7 +86,7 @@ def __init__(self, case):
         self._run_Nyears = int(stop_n)
 
         # Only allow RXCROPMATURITY to be called with test cropMonthOutput
-        if casebaseid.split("-")[-1] != "cropMonthOutput":
+        if casebaseid.split("-")[-1] != "cropMonthOutput" and not casebaseid.endswith("clm-cropMonthOutput--clm-h2a"):
             error_message = (
                 "Only call RXCROPMATURITY with test cropMonthOutput "
                 + "to avoid potentially huge sets of daily outputs."

cime_config/testdefs/testmods_dirs/clm/h2a/user_nl_clm

@@ -0,0 +1,3 @@
+
+hist_avgflag_pertape(3) = 'A'
+

python/ctsm/crop_calendars/cropcal_module.py

@@ -16,7 +16,6 @@
 
 MISSING_RX_GDD_VAL = -1
 
-
 def check_and_trim_years(year_1, year_n, ds_in):
     """
     After importing a file, restrict it to years of interest.
@@ -49,11 +48,12 @@ def check_and_trim_years(year_1, year_n, ds_in):
     return ds_in
 
 
-def open_lu_ds(filename, year_1, year_n, existing_ds, ungrid=True):
+def open_lu_ds(filename, year_1, year_n, existing_ds, logger, ungrid=True):
     """
     Open land-use dataset
     """
     # Open and trim to years of interest
+    utils.log(logger, f"open_lu_ds(): Opening this_ds_gridded: {filename}")
     this_ds_gridded = xr.open_dataset(filename).sel(time=slice(year_1, year_n))
 
     # Assign actual lon/lat coordinates
@@ -347,14 +347,15 @@ def import_output(
     gdds_rx_ds=None,
     verbose=False,
     throw_errors=True,
+    logger=None,
 ):
     """
     Import CLM output
     """
     any_bad = False
 
     # Import
-    this_ds = import_ds(filename, my_vars=my_vars, my_vegtypes=my_vegtypes)
+    this_ds = import_ds(filename, my_vars=my_vars, my_vegtypes=my_vegtypes, logger=logger)
 
     # Trim to years of interest (do not include extra year needed for finishing last growing season)
     if year_1 and year_n:

python/ctsm/crop_calendars/cropcal_utils.py

@@ -2,12 +2,42 @@
 utility functions
 copied from klindsay, https://github.com/klindsay28/CESM2_coup_carb_cycle_JAMES/blob/master/utils.py
 """
+from datetime import datetime
+import warnings
+from importlib.util import find_spec
 
 import numpy as np
 import xarray as xr
 
 from ctsm.utils import is_instantaneous
 
+with warnings.catch_warnings():
+    warnings.filterwarnings(action="ignore", category=DeprecationWarning)
+    DASK_UNAVAILABLE = find_spec("dask") is None
+
+def leading_datetime_string():
+    """
+    Return a datetime string like "YY-mm-dd HH:MM:SS    "
+    """
+    return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "    "
+
+
+def log(logger_in, string):
+    """
+    Simultaneously print INFO messages to console and to log file
+    """
+    print(leading_datetime_string() + string)
+    logger_in.info(leading_datetime_string() + string)
+
+
+def error(logger_in, string):
+    """
+    Simultaneously print ERROR messages to console and to log file
+    """
+    print(leading_datetime_string() + string)
+    logger_in.error(leading_datetime_string() + string)
+    raise RuntimeError(string)
+
 
 def define_pftlist():
     """
@@ -281,10 +311,8 @@ def vegtype_str2int(vegtype_str, vegtype_mainlist=None):
             f"Not sure how to handle vegtype_mainlist as type {type(vegtype_mainlist[0])}"
         )
 
-    if vegtype_str.shape == ():
-        indices = np.array([-1])
-    else:
-        indices = np.full(len(vegtype_str), -1)
+    vegtype_str = np.atleast_1d(vegtype_str)
+    indices = np.full(len(vegtype_str), -1)
     for vegtype_str_2 in np.unique(vegtype_str):
         indices[np.where(vegtype_str == vegtype_str_2)] = vegtype_mainlist.index(vegtype_str_2)
     if convert_to_ndarray:

python/ctsm/crop_calendars/generate_gdds.py

@@ -20,6 +20,7 @@
 sys.path.insert(1, _CTSM_PYTHON)
 import ctsm.crop_calendars.cropcal_module as cc  # pylint: disable=wrong-import-position
 import ctsm.crop_calendars.generate_gdds_functions as gddfn  # pylint: disable=wrong-import-position
+import ctsm.crop_calendars.cropcal_utils as utils  # pylint: disable=wrong-import-position
 
 # Functions here were written with too many positional arguments. At some point that should be
 # fixed. For now, we'll just disable the warning.
@@ -85,11 +86,11 @@ def main(
                 raise RuntimeError(
                     "only_make_figs True but not all plotting modules are available"
                 ) from exc
-            gddfn.log(logger, "Not all plotting modules are available; disabling save_figs")
+            utils.log(logger, "Not all plotting modules are available; disabling save_figs")
             save_figs = False
 
     # Print some info
-    gddfn.log(logger, f"Saving to {output_dir}")
+    utils.log(logger, f"Saving to {output_dir}")
 
     # Parse list of crops to skip
     if "," in skip_crops:
@@ -107,7 +108,7 @@ def main(
         yr_1_import_str = f"{first_season+1}-01-01"
         yr_n_import_str = f"{last_season+2}-01-01"
 
-        gddfn.log(
+        utils.log(
             logger,
             f"Importing netCDF time steps {yr_1_import_str} through {yr_n_import_str} "
             + "(years are +1 because of CTSM output naming)",
@@ -191,7 +192,7 @@ def main(
                 h1_instantaneous,
             )
 
-            gddfn.log(logger, f"   Saving pickle file ({pickle_file})...")
+            utils.log(logger, f"   Saving pickle file ({pickle_file})...")
             with open(pickle_file, "wb") as file:
                 pickle.dump(
                     [
@@ -219,9 +220,10 @@ def main(
             [i for i, c in enumerate(gddaccum_yp_list) if not isinstance(c, type(None))]
         ]
 
-        gddfn.log(logger, "Done")
+        utils.log(logger, "Done")
 
         if not h2_ds:
+            utils.log(logger, f"generate_gdds main(): Opening h2_ds: {h2_ds_file}")
             h2_ds = xr.open_dataset(h2_ds_file)
 
     ######################################################
@@ -236,7 +238,7 @@ def main(
             "s", sdates_rx, incl_patches1d_itype_veg, mxsowings, logger
         )
 
-        gddfn.log(logger, "Getting and gridding mean GDDs...")
+        utils.log(logger, "Getting and gridding mean GDDs...")
         gdd_maps_ds = gddfn.yp_list_to_ds(
             gddaccum_yp_list, h2_ds, incl_vegtypes_str, sdates_rx, longname_prefix, logger
         )
@@ -247,10 +249,10 @@ def main(
         # Fill NAs with dummy values
         dummy_fill = -1
         gdd_maps_ds = gdd_maps_ds.fillna(dummy_fill)
-        gddfn.log(logger, "Done getting and gridding means.")
+        utils.log(logger, "Done getting and gridding means.")
 
         # Add dummy variables for crops not actually simulated
-        gddfn.log(logger, "Adding dummy variables...")
+        utils.log(logger, "Adding dummy variables...")
         # Unnecessary?
         template_ds = xr.open_dataset(sdates_file, decode_times=True)
         all_vars = [v.replace("sdate", "gdd") for v in template_ds if "sdate" in v]
@@ -278,7 +280,7 @@ def make_dummy(this_crop_gridded, addend):
 
         for var_index, this_var in enumerate(dummy_vars):
             if this_var in gdd_maps_ds:
-                gddfn.error(
+                utils.error(
                     logger, f"{this_var} is already in gdd_maps_ds. Why overwrite it with dummy?"
                 )
             dummy_gridded.name = this_var
@@ -294,14 +296,14 @@ def add_lonlat_attrs(this_ds):
         gdd_maps_ds = add_lonlat_attrs(gdd_maps_ds)
         gddharv_maps_ds = add_lonlat_attrs(gddharv_maps_ds)
 
-        gddfn.log(logger, "Done.")
+        utils.log(logger, "Done.")
 
     ######################
     ### Save to netCDF ###
     ######################
 
     if not only_make_figs:
-        gddfn.log(logger, "Saving...")
+        utils.log(logger, "Saving...")
 
         # Get output file path
         datestr = dt.datetime.now().strftime("%Y%m%d_%H%M%S")
@@ -336,7 +338,7 @@ def save_gdds(sdates_file, hdates_file, outfile, gdd_maps_ds, sdates_rx):
 
         save_gdds(sdates_file, hdates_file, outfile, gdd_maps_ds, sdates_rx)
 
-        gddfn.log(logger, "Done saving.")
+        utils.log(logger, "Done saving.")
 
     ########################################
     ### Save things needed for mapmaking ###