Merge pull request #861 from singhd789/847.pp-an-switch-default

Update logic where `postprocess_on` and `analysis_on` is used

Author: singhd789

docs/tools/listtools.rst

@@ -2,23 +2,23 @@
 -----------------
 
 ``fre list exps [options]``
-   - Purpose: Lists available post-processing experiments included in the yaml configurations
+   - Purpose: List available post-processing experiments included in the yaml configurations
    - Options:
         - ``-y, --yamlfile [experiment yaml]``
 
 ``platforms``
 -----------------
 
 ``fre list platforms [options]``
-   - Purpose: Lists available platforms included in the yaml configurations
+   - Purpose: List available platforms included in the yaml configurations
    - Options:
         - ``-y, --yamlfile [experiment yaml]``
 
 ``pp-components``
 -----------------
 
 ``fre list pp-components [options]``
-   - Purpose: Lists components that have the `postprocess_on` key set to `True` in the postprocessing yaml configurations
+   - Purpose: List components in the postprocessing yaml that will be post-processed. All components will be post-processed unless ``postprocess_on: False`` is specified
    - Options:
         - ``-y, --yamlfile [experiment yaml]``
         - ``-e, --experiment [experiment to be post-processed]``

docs/usage/postprocess.rst

@@ -79,9 +79,9 @@ Required configuration
 
  postprocess:
    settings:
-     pp_start: 1979-01-01T0000Z
+     pp_start: "1979-01-01T0000Z"
 
-     pp_stop: 2020-01-01T0000Z
+     pp_stop: "2020-01-01T0000Z"
 
 Postprocess components
 ----------------------

docs/usage/yaml_dev/pp_yaml.rst

@@ -27,7 +27,6 @@ Required keys include:
 
     - type
     - sources
-    - postprocess_on
 
 **Settings yaml**
 
@@ -52,11 +51,8 @@ This file can follow the format below:
        pp_chunks: "array of ISO8601 datetime durations, specifying the interval of simulated time per postprocessed file" (string)
        pp_grid_spec: "path to FMS grid definition tarfile"                                                                (string)
      switches:
-       do_timeavgs: "switch to turn on/off time-average file generation"                                                  (boolean)
        clean_work: "switch to remove intermediate data files when they are no longer needed"                              (boolean)
        do_atmos_plevel_masking: "switch to mask atmos pressure-level output above/below surface pressure/atmos top"       (boolean)
-       do_analysis: "switch to launch analysis scripts"                                                                   (boolean)
-       do_analysis_only: "switch to only launch analysis scripts"                                                         (boolean)
 
 Required keys include:
 
@@ -72,5 +68,3 @@ Required keys include:
     - clean_work
     - do_timeavgs
     - do_atmos_plevel_masking
-    - do_analysis
-    - do_analysis_only

fre/app/regrid_xy/regrid_xy.py

@@ -342,12 +342,14 @@ def regrid_xy(yamlfile: str,
 
         # submit fregrid job for each component
         for component in components:
-
-            # skip component if postprocess_on = False
-            if not component["postprocess_on"]:
-                fre_logger.warning(f"postprocess_on=False for {source} in component {component['type']}." \
-                                    "Skipping {source}")
-                continue
+            # If postprocess_on is not defined, it should have the default value of True
+            # If postprocess_on is defined, check for a True or False value
+            if "postprocess_on" in component:
+                # skip component if postprocess_on = False
+                if not component["postprocess_on"]:
+                    fre_logger.warning(f"postprocess_on=False for {source} in component {component['type']}." \
+                                        "Skipping {source}")
+                    continue
 
             # skip component if xyInterp is not set
             if 'xyInterp' not in component:

fre/gfdl_msd_schemas

@@ -44,7 +44,10 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str):
     # log the experiment names, which should show up on screen for sure
     fre_logger.info("Components to be post-processed:")
     for i in yml_dict["postprocess"]["components"]:
-        if i.get("postprocess_on"):
+        if "postprocess_on" in i:
+            if i.get("postprocess_on") is True:
+                fre_logger.info('   - %s', i.get("type"))
+        else:
             fre_logger.info('   - %s', i.get("type"))
     fre_logger.info("\n")
 

fre/list_/list_pp_components_script.py

@@ -2,13 +2,7 @@
 Test fre list pp-comps
 """
 from pathlib import Path
-
-import pytest
-import yaml
-
 from fre.list_ import list_pp_components_script
-from fre.yamltools import combine_yamls_script as cy
-from fre.yamltools import helpers
 
 
 # SET-UP
@@ -37,13 +31,18 @@ def test_ppyamls_exist():
         assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{pp_yaml}").exists()
 
 # Test whole tool
-def test_exp_list(caplog):
+def test_pp_comp_list(caplog):
     ''' Test fre list pp-components subtool '''
     list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME)
 
     # check the logging output
     check_out = [ 'Components to be post-processed:',
+                  '   - atmos_cmip',
                   '   - atmos',
+                  '   - atmos_level_cmip',
+                  '   - atmos_level',
+                  '   - atmos_month_aer',
+                  '   - atmos_diurnal',
                   '   - atmos_scalar']
 
     for i in check_out:
@@ -66,4 +65,4 @@ def test_yamlvalidate(caplog):
         assert i in caplog.text
 
     for record in caplog.records:
-        record.levelname == "INFO"
+        assert record.levelname == "INFO"

fre/list_/tests/test_list_pp_components_script.py

@@ -9,5 +9,3 @@ postprocess:
     site:               "ppan"
   switches:
     clean_work:         True
-    do_refinediag:      False
-    do_analysis:        True

fre/make/tests/null_example/settings.yaml

@@ -10,28 +10,3 @@ fre_properties:
 build:
   compileYaml:  "../compile.yaml"
   platformYaml: "empty_platforms.yaml"
-
-shared:                                                                                                        # directories shared across tools
-  directories: &shared_directories
-    history_dir:  !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
-    pp_dir:       !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
-    analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
-
-  # shared pp settings
-  postprocess:
-    settings: &shared_settings
-      site:               "ppan"
-    switches: &shared_switches
-      clean_work:         True
-      do_refinediag:      False
-      do_analysis:        True
-
-experiments:
-  - name: "null_model_full"
-    pp:
-  - name: "null_model_0"
-    pp:
-  - name: "null_model_1"
-    pp:
-  - name: "null_model_2"
-    pp:

fre/make/tests/null_example/wrong_model/wrong_null_model.yaml

@@ -122,6 +122,7 @@ def set_rose_suite(yamlfile: dict, rose_suite: metomi.rose.config.ConfigNode) ->
     """
     pp=yamlfile.get("postprocess")
     dirs=yamlfile.get("directories")
+    analysis=yamlfile.get("analysis")
 
     # set rose-suite items
     pa_scripts = ""
@@ -186,6 +187,34 @@ def set_rose_suite(yamlfile: dict, rose_suite: metomi.rose.config.ConfigNode) ->
         rose_suite.set( keys = ['template variables', 'DO_PREANALYSIS'],
                         value = 'False' )
 
+    # Set DO_ANALYSIS switch
+    # analysis_on is optional key for each component in the analysis yaml and 
+    # defaults to True if not specified.
+    # In the rose_suite.conf:
+    #  - if 'analysis_on: False' for all analysis components, set DO_ANALYSIS=False
+    #  - if 'analysis_on: True' for any analysis components, set DO_ANALYSIS=True
+    do_analysis_switch = []
+    if not analysis:
+        return
+        
+    for an_key, an_value in analysis.items():
+        an_workflow_info = an_value["workflow"]
+        # if analysis_on key is actually set, evaluate and save its value in a list
+        if "analysis_on" in an_workflow_info:
+            do_analysis_switch.append(an_workflow_info["analysis_on"])
+        #if analysis_on key is NOT set, save its value as True in the list
+        else:
+            do_analysis_switch.append("True")
+
+    # if ANY of the analysis components do not set analysis_on or set analysis_on: True,
+    # set DO_ANALYSIS=True in the rose_suite.conf
+    if any(do_analysis_switch):
+        rose_suite.set( keys = ['template variables', 'DO_ANALYSIS'],
+                        value = 'True' )
+    else:
+        rose_suite.set( keys = ['template variables', 'DO_ANALYSIS'],
+                        value = 'False' )
+
     if dirs is not None:
         for key,value in dirs.items():
             rose_suite.set(keys=['template variables', key.upper()], value=quote_rose_values(value))