Add precip_pdf and mp_partition from e3sm diags

examples/post.v3.LR.historical.zppy_v3.cfg

@@ -1,5 +1,5 @@
-# This is an example zppy v3.1.0 configuration file for Chrysalis.
-# zppy v3.1.0 is included in E3SM Unified 1.12.
+# This is an example zppy v3.2.0 configuration file for Chrysalis.
+# zppy v3.2.0 is included in E3SM Unified 1.13.
 #
 # NOTE: This configuration generates approximately 220 tasks/jobs when fully enabled.
 # Users should only enable the tasks and time periods they need based on their specific
@@ -8,15 +8,15 @@
 [default]
 # Change these two paths to include your username instead
 # You can also change these to match your preferred directory structure
-output = /lcrc/group/e3sm/ac.zhang40/zppy_example_v3.1.0/v3.LR.historical_0051
-www = /lcrc/group/e3sm/public_html/diagnostic_output/zppy_example/v3.1.0
+output = /lcrc/group/e3sm/ac.zhang40/zppy_example_v3.2.0/v3.LR.historical_0051
+www = /lcrc/group/e3sm/public_html/diagnostic_output/zppy_example/v3.2.0
 # Other paths
 input = /lcrc/group/e3sm2/ac.wlin/E3SMv3/v3.LR.historical_0051
 
 # zppy auto-determines and sources latest e3sm-unified on each supported machine with environment_commands commented out.
 #environment_commands = "source /lcrc/soft/climate/e3sm-unified/load_latest_e3sm_unified_chrysalis.sh"
 # For testing Unified rc:
-#environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
+#environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.13.0rc2_chrysalis.sh"
 # Other parameters:
 campaign = "water_cycle"
 case = v3.LR.historical_0051
@@ -67,15 +67,15 @@ years = "1850:2014:5",
   [[ atm_monthly_180x360_aave ]]
   # This subtask is a dependency for the e3sm_diags task's atm_monthly_180x360 and atm_monthly_180x360_aave_mvm subtasks.
   # The following e3sm_diags sets require it:
-  # "enso_diags", "qbo", "area_mean_time_series"
+  # "enso_diags", "qbo", "area_mean_time_series", "mp_partition"
   # This subtask is also a dependency for the e3sm_to_cmip task's atm_monthly_180x360_aave subtask.
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   mapping_file = map_ne30pg2_to_cmip6_180x360_aave.20200201.nc
-  vars = "AODABS,AODALL,AODBC,AODDUST,AODPOM,AODSO4,AODSOA,AODSS,AODVIS,AREL,CLDHGH,CLDLOW,CLDMED,CLDTOT,CO_SRF,FLDS,FLNS,FLNSC,FLNT,FLUT,FLUTC,FSDS,FSDSC,FSNS,FSNSC,FSNT,FSNTOA,FSNTOAC,ICEFRAC,LANDFRAC,LHFLX,OCNFRAC,PRECC,PRECL,PRECSC,PRECSL,PS,PSL,QFLX,QREFHT,SCO,SHFLX,SOLIN,TAUX,TAUY,TCO,TGCLDCWP,TGCLDIWP,TMQ,TREFHT,TREFMNAV,TREFMXAV,TROP_P,TS,U10,U,T,O3,CLDICE,CLDLIQ"
-  # Needed for mixed-phase partition:
-  #  vars = "LANDFRAC,CLDICE,CLDLIQ,T"
+  vars = "AODABS,AODALL,AODBC,AODDUST,AODPOM,AODSO4,AODSOA,AODSS,AODVIS,AREL,CLDHGH,CLDLOW,CLDMED,CLDTOT,CO_SRF,FLDS,FLNS,FLNSC,FLNT,FLUT,FLUTC,FSDS,FSDSC,FSNS,FSNSC,FSNT,FSNTOA,FSNTOAC,ICEFRAC,LANDFRAC,LHFLX,OCNFRAC,PRECC,PRECL,PRECSC,PRECSL,PS,PSL,QFLX,QREFHT,SCO,SHFLX,SOLIN,TAUX,TAUY,TCO,TGCLDCWP,TGCLDIWP,TMQ,TREFHT,TREFMNAV,TREFMXAV,TROP_P,TS,U10,U,T,O3,CLDICE,CLDLIQ,CLD_CAL_TMPICE,CLD_CAL_TMPLIQ"
+  # Needed for mixed-phase partition (included above):
+  # LANDFRAC, CLD_CAL_TMPICE, CLD_CAL_TMPLIQ or CLDICE, CLDLIQ, T
 
   [[ land_monthly ]]
   # This subtask is a dependency for the e3sm_to_cmip task's land_monthly subtask.
@@ -90,7 +90,7 @@ years = "1850:2014:5",
   [[ atm_daily_180x360_aave ]]
   # This subtask is a dependency for the e3sm_diags task's atm_monthly_180x360 subtask.
   # The following e3sm_diags sets require it:
-  # "tropical_subseasonal"
+  # "tropical_subseasonal", "precip_pdf"
   frequency = "daily"
   input_files = "eam.h1"
   input_subdir = "archive/atm/hist"
@@ -168,13 +168,13 @@ ref_start_yr = 1985
 ts_num_years = 5
 walltime = "4:00:00"
 years = "1985:2014:30",
+# NOTE: If you want to use the latest development environment of e3sm_diags,
+# you can do something like the following:
+environment_commands = "source /home/ac.zhang40/y/etc/profile.d/conda.sh; conda activate e3sm_diags_dev_py313"
 
   [[ atm_monthly_180x360_aave ]]
-  # NOTE: If you want to use the latest development environment of e3sm_diags,
-  # you can do something like the following:
-  # environment_commands = "source /home/ac.zhang40/y/etc/profile.d/conda.sh; conda activate e3sm_diags_dev"
   # `e3sm_diags` is largely driven by which e3sm_diags sets are requested:
-  sets="lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","annual_cycle_zonal_mean","enso_diags","qbo","diurnal_cycle","zonal_mean_2d_stratosphere","aerosol_aeronet","tropical_subseasonal","tc_analysis", "tropical_subseasonal","streamflow",
+  sets="lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","annual_cycle_zonal_mean","enso_diags","qbo","diurnal_cycle","zonal_mean_2d_stratosphere","aerosol_aeronet","mp_partition","tropical_subseasonal","precip_pdf","tc_analysis","streamflow",
   # We can categorize e3sm_diags sets into 6 groups, based on dependencies.
   # Different e3sm_diags sets require different parameters to be set.
   #
@@ -186,9 +186,9 @@ years = "1985:2014:30",
   # The name of this subtask doesn't match up with the name of the subtask for diurnal data in the `climo` task. We therefore need to explicitly tell zppy which subtask to use.
   climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
   # 3. These sets depend on the ts task's atm_monthly_180x360_aave subtask:
-  # "enso_diags" , "qbo", "area_mean_time_series" (not requested)
+  # "enso_diags" , "qbo", "area_mean_time_series" (not requested), "mp_partition"
   # 4. These sets depend on the ts task's atm_daily_180x360_aave subtask:
-  # "tropical_subseasonal"
+  # "tropical_subseasonal", "precip_pdf"
   ts_daily_subsection = "atm_daily_180x360_aave"
   # 5. These sets depend on the ts task's rof_monthly subtask:
   # "streamflow"
@@ -202,6 +202,7 @@ years = "1985:2014:30",
   # Save subplot in pdf format
   # output_format_subplot = "pdf",
 
+  #Model vs model tasks requires reference data already being processed by zppy, example paths shown as below
   [[atm_monthly_180x360_aave_mvm]]
   sets="lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","annual_cycle_zonal_mean","qbo","diurnal_cycle","zonal_mean_2d_stratosphere",
   # Using the groups from above:
@@ -294,5 +295,5 @@ nodes = 1
 short_name = 'v3.LR.historical_0051'
 ts_num_years = 5
 walltime = "2:00:00"
-cfg = "inclusions/ilamb/cmip.cfg"
+cfg = "inclusions/ilamb/ilamb.cfg"
 years = "1985:2014:30"

tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg

@@ -49,6 +49,9 @@ walltime = "00:30:00"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   years = "1985:1995:2", # Need 10 years for pcmdi_diags task
+  # We need to add extra vars to the default `vars=""` value
+  # in order to enable the e3sm_diags mp_partition set.
+  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,PSL,LANDFRAC,CLD_CAL_TMPICE,CLD_CAL_TMPLIQ,CLDLIQ,T"
 
   [[ atm_daily_180x360_aave ]]
   active = True
@@ -158,6 +161,7 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/"
   dc_obs_climo = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
   # mvo streamflow only
   streamflow_obs_ts = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/"
+  sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","annual_cycle_zonal_mean","zonal_mean_2d_stratosphere","enso_diags","qbo","diurnal_cycle","streamflow","mp_partition","tropical_subseasonal","precip_pdf","aerosol_aeronet",
 
   [[ atm_monthly_180x360_aave_mvm ]]
   # Test model-vs-model using the same files as the reference

tests/integration/template_weekly_comprehensive_v3.cfg

@@ -49,6 +49,9 @@ walltime = "00:30:00"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   years = "1985:1995:2", # Need 10 years for pcmdi_diags task
+  # We need to add extra vars to the default `vars=""` value
+  # in order to enable the e3sm_diags mp_partition set.
+  vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,PSL,LANDFRAC,CLD_CAL_TMPICE,CLD_CAL_TMPLIQ,CLDLIQ,T"
 
   [[ atm_daily_180x360_aave ]]
   active = #expand active_e3sm_diags#
@@ -158,6 +161,7 @@ tc_obs = "#expand diagnostics_base_path#/observations/Atm/tc-analysis/"
   dc_obs_climo = '#expand path_dc_obs_climo#'
   # mvo streamflow only
   streamflow_obs_ts = "#expand diagnostics_base_path#/observations/Atm/time-series/"
+  sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","annual_cycle_zonal_mean","zonal_mean_2d_stratosphere","enso_diags","qbo","diurnal_cycle","streamflow","mp_partition","tropical_subseasonal","precip_pdf","aerosol_aeronet",
 
   [[ atm_monthly_180x360_aave_mvm ]]
   # Test model-vs-model using the same files as the reference

tests/test_zppy_e3sm_diags.py

@@ -189,6 +189,38 @@ def test_check_mvm_only_parameters_for_bash():
     with pytest.raises(ParameterNotProvidedError):
         check_mvm_only_parameters_for_bash(c)
 
+    # mp_partition
+    c = {"sets": ["mp_partition"]}
+    c.update(d0)
+    check_mvm_only_parameters_for_bash(c)
+    c.update(d1)
+    check_mvm_only_parameters_for_bash(c)  # ref_final_yr not needed
+    c.update(d2)
+    check_mvm_only_parameters_for_bash(c)  # ref_start_yr not needed
+    c.update(d3)
+    with pytest.raises(ParameterNotProvidedError):
+        check_mvm_only_parameters_for_bash(c)
+    c.update(d4)
+    with pytest.raises(ParameterNotProvidedError):
+        check_mvm_only_parameters_for_bash(c)
+
+    # precip_pdf
+    c = {"sets": ["precip_pdf"]}
+    c.update(d0)
+    check_mvm_only_parameters_for_bash(c)
+    c.update(d1)
+    with pytest.raises(ParameterNotProvidedError):
+        check_mvm_only_parameters_for_bash(c)
+    c.update(d2)
+    with pytest.raises(ParameterNotProvidedError):
+        check_mvm_only_parameters_for_bash(c)
+    c.update(d3)
+    with pytest.raises(ParameterNotProvidedError):
+        check_mvm_only_parameters_for_bash(c)
+    c.update(d4)
+    with pytest.raises(ParameterNotProvidedError):
+        check_mvm_only_parameters_for_bash(c)
+
 
 def test_check_and_define_parameters():
     # test_zppy_utils.py tests the inference functionality turned off.
@@ -286,7 +318,7 @@ def test_check_and_define_parameters():
     assert c["obs_ts"] == "diags/post/observations/Atm/time-series/"
 
     # area_mean_time_series/enso_diags/qbo, mvm
-    for diags_set in ["area_mean_time_series", "enso_diags", "qbo"]:
+    for diags_set in ["area_mean_time_series", "enso_diags", "qbo", "mp_partition"]:
         c = {
             "sets": [diags_set],
             "run_type": "model_vs_model",
@@ -455,28 +487,29 @@ def test_check_and_define_parameters():
     assert c["tc_obs"] == "diags/post/observations/Atm/tc-analysis/"
     assert c["reference_data_path_tc"] == "diags/post/atm/tc-analysis_1980_1990"
 
-    # tropical_subseasonal, mvm
-    c = {
-        "sets": ["tropical_subseasonal"],
-        "run_type": "model_vs_model",
-        "reference_data_path": "",
-        "reference_data_path_ts_daily": "a",
-        "grid": "grid",
-    }
-    c.update(mvm_base)
-    check_and_define_parameters(c)
-    assert c["reference_data_path_ts_daily"] == "a"
+    # tropical_subseasonal/precip_pdf, mvm
+    for diags_set in ["tropical_subseasonal", "precip_pdf"]:
+        c = {
+            "sets": [diags_set],
+            "run_type": "model_vs_model",
+            "reference_data_path": "",
+            "reference_data_path_ts_daily": "a",
+            "grid": "grid",
+        }
+        c.update(mvm_base)
+        check_and_define_parameters(c)
+        assert c["reference_data_path_ts_daily"] == "a"
 
-    c = {
-        "sets": ["tropical_subseasonal"],
-        "run_type": "model_vs_model",
-        "reference_data_path": "",
-        "reference_data_path_ts_daily": "",
-        "grid": "grid",
-    }
-    c.update(mvm_base)
-    check_and_define_parameters(c)
-    assert c["reference_data_path_ts_daily"] == "diags/post/atm/grid/ts/daily"
+        c = {
+            "sets": [diags_set],
+            "run_type": "model_vs_model",
+            "reference_data_path": "",
+            "reference_data_path_ts_daily": "",
+            "grid": "grid",
+        }
+        c.update(mvm_base)
+        check_and_define_parameters(c)
+        assert c["reference_data_path_ts_daily"] == "diags/post/atm/grid/ts/daily"
 
 
 def test_add_climo_dependencies():
@@ -523,7 +556,7 @@ def test_add_ts_dependencies():
         "ts_subsection": "sub",
         "ts_daily_subsection": "dsub",
     }
-    sets = ["area_mean_time_series", "enso_diags", "qbo"]
+    sets = ["area_mean_time_series", "enso_diags", "qbo", "mp_partition"]
     for diags_set in sets:
         c: Dict[str, Any] = {"sets": [diags_set]}
         c.update(base)
@@ -539,8 +572,9 @@ def test_add_ts_dependencies():
     add_ts_dependencies(c, dependencies, "script_dir", 1980)
     assert dependencies == ["script_dir/ts_rof_monthly_1980-1984-0005.status"]
 
-    c = {"sets": ["tropical_subseasonal"]}
-    c.update(base)
-    dependencies = []
-    add_ts_dependencies(c, dependencies, "script_dir", 1980)
-    assert dependencies == ["script_dir/ts_dsub_1980-1984-0005.status"]
+    for diags_set in ["tropical_subseasonal", "precip_pdf"]:
+        c = {"sets": [diags_set]}
+        c.update(base)
+        dependencies = []
+        add_ts_dependencies(c, dependencies, "script_dir", 1980)
+        assert dependencies == ["script_dir/ts_dsub_1980-1984-0005.status"]

zppy/__main__.py

@@ -37,6 +37,17 @@ def main():
     template_dir: str = os.path.join(os.path.dirname(__file__), "templates")
     # Subdirectory where defaults are located
     defaults_dir: str = os.path.join(os.path.dirname(__file__), "defaults")
+    # Warn if templateDir is outside the active conda environment. This can
+    # happen when PYTHONPATH points to a local zppy clone and takes precedence
+    # over the conda env's site-packages.
+    conda_prefix: str = os.environ.get("CONDA_PREFIX", "")
+    if conda_prefix and not template_dir.startswith(conda_prefix):
+        logger.warning(
+            f"templateDir '{template_dir}' is outside the active conda environment "
+            f"'{conda_prefix}'. This is likely caused by PYTHONPATH including a local "
+            f"zppy clone. Consider running 'unset PYTHONPATH' before activating the "
+            f"conda environment."
+        )
     # Read configuration file and validate it
     default_config: str = os.path.join(defaults_dir, "default.ini")
     user_config: ConfigObj = ConfigObj(args.config, configspec=default_config)

zppy/defaults/default.ini

@@ -247,7 +247,7 @@ reference_data_path_ts_rof = string(default="")
 # Search for `Required for run_type="model_vs_model"` in this file.
 run_type = string(default="model_vs_obs")
 # The sets to run
-# All available sets (17) = "aerosol_aeronet","aerosol_budget","annual_cycle_zonal_mean","area_mean_time_series","cosp_histogram","diurnal_cycle","enso_diags","lat_lon","meridional_mean_2d","polar","qbo","streamflow","tc_analysis", "tropical_subseasonal", "zonal_mean_2d","zonal_mean_2d_stratosphere","zonal_mean_xy"
+# All available sets (19) = "aerosol_aeronet","aerosol_budget","annual_cycle_zonal_mean","area_mean_time_series","cosp_histogram","diurnal_cycle","enso_diags","lat_lon","meridional_mean_2d","mp_partition","polar","precip_pdf","qbo","streamflow","tc_analysis","tropical_subseasonal","zonal_mean_2d","zonal_mean_2d_stratosphere","zonal_mean_xy"
 # To find the parameters required for a set, search for the set's name in this file.
 # The order of the `sets` list is the order the sets will show up in E3SM Diags.
 # `sets` below are ordered by 1) core or speciality and then 2) older to newer.

zppy/e3sm_diags.py

@@ -120,18 +120,30 @@ def check_mvm_only_parameters_for_bash(c: Dict[str, Any]) -> None:
 
     check_set_specific_parameter(
         c,
-        set(["enso_diags", "tropical_subseasonal", "streamflow", "tc_analysis"]),
+        set(
+            [
+                "enso_diags",
+                "tropical_subseasonal",
+                "precip_pdf",
+                "streamflow",
+                "tc_analysis",
+            ]
+        ),
         "ref_final_yr",
     )
     check_set_specific_parameter(
-        c, set(["tropical_subseasonal", "streamflow", "tc_analysis"]), "ref_start_yr"
+        c,
+        set(["tropical_subseasonal", "precip_pdf", "streamflow", "tc_analysis"]),
+        "ref_start_yr",
     )
     ts_sets = set(
         [
             "enso_diags",
             "qbo",
             "area_mean_time_series",
+            "mp_partition",
             "tropical_subseasonal",
+            "precip_pdf",
             "streamflow",
         ]
     )
@@ -198,14 +210,16 @@ def check_and_define_parameters(c: Dict[str, Any]) -> None:
             c["reference_data_path_tc"] = (
                 f"{reference_data_path}/atm/tc-analysis_{c['ref_year1']}_{c['ref_year2']}"
             )
-        if set(["enso_diags", "qbo", "area_mean_time_series"]) & set(c["sets"]):
+        if set(["enso_diags", "qbo", "area_mean_time_series", "mp_partition"]) & set(
+            c["sets"]
+        ):
             set_value_of_parameter_if_undefined(
                 c,
                 "reference_data_path_ts",
                 f"{reference_data_path}/atm/{c['grid']}/ts/monthly",
                 ParameterInferenceType.PATH_INFERENCE,
             )
-        if "tropical_subseasonal" in c["sets"]:
+        if set(["tropical_subseasonal", "precip_pdf"]) & set(c["sets"]):
             set_value_of_parameter_if_undefined(
                 c,
                 "reference_data_path_ts_daily",
@@ -288,7 +302,9 @@ def add_ts_dependencies(
     ts_daily_sub = get_value_from_parameter(
         c, "ts_daily_subsection", "sub", ParameterInferenceType.SECTION_INFERENCE
     )
-    depend_on_ts: Set[str] = set(["enso_diags", "qbo", "area_mean_time_series"])
+    depend_on_ts: Set[str] = set(
+        ["enso_diags", "qbo", "area_mean_time_series", "mp_partition"]
+    )
     if depend_on_ts & set(c["sets"]):
         # ts task
         add_dependencies(
@@ -310,7 +326,7 @@ def add_ts_dependencies(
             end_yr,
             c["ts_num_years"],
         )
-    if "tropical_subseasonal" in c["sets"]:
+    if set(["tropical_subseasonal", "precip_pdf"]) & set(c["sets"]):
         add_dependencies(
             dependencies,
             script_dir,