Fixes for rc2 (#748)

* Updates from Unified testing

* Improve error handling in templates

* Add parameter inference notes

Author: forsyth2

tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v3_chrysalis.cfg

@@ -4,7 +4,12 @@ constraint = ""
 dry_run = "False"
 environment_commands = ""
 fail_on_dependency_skip = True
-infer_path_parameters = False
+# This setting does not support backwards compatibility.
+# Its purpose is specifically to call attention to parameters
+# that zppy would otherwise just infer, perhaps making an incorrect inference.
+# Therefore, it is expected that a user using this setting would want to get an error
+# noting new parameters to add explicitly to their cfg.
+# infer_path_parameters = False
 infer_section_parameters = False
 input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051
 input_subdir = archive/atm/hist

tests/integration/template_weekly_legacy_3.0.0_comprehensive_v3.cfg

@@ -4,7 +4,12 @@ constraint = "#expand constraint#"
 dry_run = "#expand dry_run#"
 environment_commands = "#expand environment_commands#"
 fail_on_dependency_skip = True
-infer_path_parameters = False
+# This setting does not support backwards compatibility.
+# Its purpose is specifically to call attention to parameters
+# that zppy would otherwise just infer, perhaps making an incorrect inference.
+# Therefore, it is expected that a user using this setting would want to get an error
+# noting new parameters to add explicitly to their cfg.
+# infer_path_parameters = False
 infer_section_parameters = False
 input = #expand user_input_v3#/E3SMv3/#expand case_name#
 input_subdir = archive/atm/hist

tests/integration/utils.py

@@ -127,7 +127,7 @@ def get_expansions():
     # Set up environments
     # To run this test, replace conda environment with your e3sm_diags dev environment
     # Or the Unified environment
-    # (The same for `global_time_series_environment_commands`)
+    # (The same for `global_time_series_environment_commands` and `pcmdi_diags_environment_commands`)
     # Never set this to "" because it will print the line
     # `environment_commands = ""` for the [e3sm_diags] task, overriding zppy's
     # default of using Unified. That is, there will be no environment set.
@@ -203,7 +203,7 @@ def generate_cfgs(unified_testing=False, dry_run=False):
     expansions = get_expansions()
     if unified_testing:
         expansions["environment_commands"] = expansions["environment_commands_test"]
-        # Use Unified for e3sm_diags and global_time_series unless we specify otherwise
+        # Use Unified for e3sm_diags, global_time_series, pcmdi_diags unless we specify otherwise
         if expansions["diags_environment_commands"] == "":
             expansions["diags_environment_commands"] = expansions[
                 "environment_commands_test"
@@ -212,6 +212,10 @@ def generate_cfgs(unified_testing=False, dry_run=False):
             expansions["global_time_series_environment_commands"] = expansions[
                 "environment_commands_test"
             ]
+        if expansions["pcmdi_diags_environment_commands"] == "":
+            expansions["pcmdi_diags_environment_commands"] = expansions[
+                "environment_commands_test"
+            ]
     else:
         # The cfg doesn't need this line,
         # but it would be difficult to only write environment_commands in the unified_testing case.

zppy/defaults/default.ini

@@ -31,6 +31,9 @@ grid = string(default="")
 # These two parameters enable zppy to infer path or section parameters.
 # This allows users to set fewer parameters, but with the risk of zppy choosing incorrect values for them.
 # Set to False for more transparency in path or section defintions.
+# Please note `infer_path_parameters = False` does NOT support backwards compatibility.
+# The expectation is the user will want an error message describing parameters they haven't explicitly defined.
+# So, as new parameters get added in successive zppy versions, they can update their cfgs accordingly.
 infer_path_parameters = boolean(default=True)
 infer_section_parameters = boolean(default=True)
 # The directory to be post-processed

zppy/templates/climo.bash

@@ -1,7 +1,9 @@
 #!/bin/bash
 {% include 'inclusions/slurm_header.bash' %}
 {% include 'inclusions/boilerplate.bash' %}
+set -e
 {{ environment_commands }}
+set +e
 
 # Create temporary workdir
 hash=`mktemp --dry-run -d XXXX`

zppy/templates/e3sm_diags.bash

@@ -1,7 +1,9 @@
 #!/bin/bash
 {% include 'inclusions/slurm_header.bash' %}
 {% include 'inclusions/boilerplate.bash' %}
+set -e
 {{ environment_commands }}
+set +e
 
 # Make sure UVCDAT doesn't prompt us about anonymous logging
 export UVCDAT_ANONYMOUS_LOG=False

zppy/templates/e3sm_to_cmip.bash

@@ -1,7 +1,9 @@
 #!/bin/bash
 {% include 'inclusions/slurm_header.bash' %}
 {% include 'inclusions/boilerplate.bash' %}
+set -e
 {{ environment_commands }}
+set +e
 
 # Create temporary workdir
 hash=`mktemp --dry-run -d XXXX`

zppy/templates/global_time_series.bash

@@ -1,7 +1,9 @@
 #!/bin/bash
 {% include 'inclusions/slurm_header.bash' %}
 {% include 'inclusions/boilerplate.bash' %}
+set -e
 {{ environment_commands }}
+set +e
 
 # Generate global time series plots
 ################################################################################

zppy/templates/ilamb.bash

@@ -1,7 +1,9 @@
 #!/bin/bash
 {% include 'inclusions/slurm_header.bash' %}
 {% include 'inclusions/boilerplate.bash' %}
+set -e
 {{ environment_commands }}
+set +e
 
 # Point to observation data
 export ILAMB_ROOT={{ ilamb_obs }}

zppy/templates/mpas_analysis.bash

@@ -1,7 +1,9 @@
 #!/bin/bash
 {% include 'inclusions/slurm_header.bash' %}
 {% include 'inclusions/boilerplate.bash' %}
+set -e
 {{ environment_commands }}
+set +e
 
 # Additional settings for MPAS-Analysis
 export OMP_NUM_THREADS=1