Update logic where postprocess_on and analysis_on is used

docs/tools/listtools.rst

@@ -2,23 +2,23 @@
 -----------------
 
 ``fre list exps [options]``
-   - Purpose: Lists available post-processing experiments included in the yaml configurations
+   - Purpose: List available post-processing experiments included in the yaml configurations
    - Options:
         - ``-y, --yamlfile [experiment yaml]``
 
 ``platforms``
 -----------------
 
 ``fre list platforms [options]``
-   - Purpose: Lists available platforms included in the yaml configurations
+   - Purpose: List available platforms included in the yaml configurations
    - Options:
         - ``-y, --yamlfile [experiment yaml]``
 
 ``pp-components``
 -----------------
 
 ``fre list pp-components [options]``
-   - Purpose: Lists components that have the `postprocess_on` key set to `True` in the postprocessing yaml configurations
+   - Purpose: List components in the postprocessing yaml that will be post-processed. All components will be post-processed unless ``postprocess_on: False`` is specified
    - Options:
         - ``-y, --yamlfile [experiment yaml]``
         - ``-e, --experiment [experiment to be post-processed]``

docs/usage/postprocess.rst

@@ -79,9 +79,9 @@ Required configuration
 
  postprocess:
    settings:
-     pp_start: 1979-01-01T0000Z
+     pp_start: "1979-01-01T0000Z"
 
-     pp_stop: 2020-01-01T0000Z
+     pp_stop: "2020-01-01T0000Z"
 
 Postprocess components
 ----------------------

docs/usage/yaml_dev/pp_yaml.rst

@@ -27,7 +27,6 @@ Required keys include:
 
     - type
     - sources
-    - postprocess_on
 
 **Settings yaml**
 
@@ -52,11 +51,8 @@ This file can follow the format below:
        pp_chunks: "array of ISO8601 datetime durations, specifying the interval of simulated time per postprocessed file" (string)
        pp_grid_spec: "path to FMS grid definition tarfile"                                                                (string)
      switches:
-       do_timeavgs: "switch to turn on/off time-average file generation"                                                  (boolean)
        clean_work: "switch to remove intermediate data files when they are no longer needed"                              (boolean)
        do_atmos_plevel_masking: "switch to mask atmos pressure-level output above/below surface pressure/atmos top"       (boolean)
-       do_analysis: "switch to launch analysis scripts"                                                                   (boolean)
-       do_analysis_only: "switch to only launch analysis scripts"                                                         (boolean)
 
 Required keys include:
 
@@ -72,5 +68,3 @@ Required keys include:
     - clean_work
     - do_timeavgs
     - do_atmos_plevel_masking
-    - do_analysis
-    - do_analysis_only

fre/app/regrid_xy/regrid_xy.py

@@ -342,12 +342,14 @@ def regrid_xy(yamlfile: str,
 
         # submit fregrid job for each component
         for component in components:
-
-            # skip component if postprocess_on = False
-            if not component["postprocess_on"]:
-                fre_logger.warning(f"postprocess_on=False for {source} in component {component['type']}." \
-                                    "Skipping {source}")
-                continue
+            # If postprocess_on is not defined, it should have the default value of True
+            # If postprocess_on is defined, check for a True or False value
+            if "postprocess_on" in component:
+                # skip component if postprocess_on = False
+                if not component["postprocess_on"]:
+                    fre_logger.warning(f"postprocess_on=False for {source} in component {component['type']}." \
+                                        "Skipping {source}")
+                    continue
 
             # skip component if xyInterp is not set
             if 'xyInterp' not in component:

fre/list_/list_pp_components_script.py

@@ -44,7 +44,10 @@ def list_ppcomps_subtool(yamlfile: str, experiment: str):
     # log the experiment names, which should show up on screen for sure
     fre_logger.info("Components to be post-processed:")
     for i in yml_dict["postprocess"]["components"]:
-        if i.get("postprocess_on"):
+        if "postprocess_on" in i:
+            if i.get("postprocess_on") is True:
+                fre_logger.info('   - %s', i.get("type"))
+        else:
             fre_logger.info('   - %s', i.get("type"))
     fre_logger.info("\n")
 

fre/list_/tests/test_list_pp_components_script.py

@@ -2,13 +2,7 @@
 Test fre list pp-comps
 """
 from pathlib import Path
-
-import pytest
-import yaml
-
 from fre.list_ import list_pp_components_script
-from fre.yamltools import combine_yamls_script as cy
-from fre.yamltools import helpers
 
 
 # SET-UP
@@ -37,13 +31,18 @@ def test_ppyamls_exist():
         assert Path(f"{TEST_DIR}/{AM5_EXAMPLE}/{pp_yaml}").exists()
 
 # Test whole tool
-def test_exp_list(caplog):
+def test_pp_comp_list(caplog):
     ''' Test fre list pp-components subtool '''
     list_pp_components_script.list_ppcomps_subtool(f"{TEST_DIR}/{AM5_EXAMPLE}/{MODEL_YAMLFILE}", EXP_NAME)
 
     # check the logging output
     check_out = [ 'Components to be post-processed:',
+                  '   - atmos_cmip',
                   '   - atmos',
+                  '   - atmos_level_cmip',
+                  '   - atmos_level',
+                  '   - atmos_month_aer',
+                  '   - atmos_diurnal',
                   '   - atmos_scalar']
 
     for i in check_out:
@@ -66,4 +65,4 @@ def test_yamlvalidate(caplog):
         assert i in caplog.text
 
     for record in caplog.records:
-        record.levelname == "INFO"
+        assert record.levelname == "INFO"

fre/make/tests/null_example/settings.yaml

@@ -9,5 +9,4 @@ postprocess:
     site:               "ppan"
   switches:
     clean_work:         True
-    do_refinediag:      False
     do_analysis:        True

fre/make/tests/null_example/wrong_model/wrong_null_model.yaml

@@ -10,28 +10,3 @@ fre_properties:
 build:
   compileYaml:  "../compile.yaml"
   platformYaml: "empty_platforms.yaml"
-
-shared:                                                                                                        # directories shared across tools
-  directories: &shared_directories
-    history_dir:  !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
-    pp_dir:       !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
-    analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
-
-  # shared pp settings
-  postprocess:
-    settings: &shared_settings
-      site:               "ppan"
-    switches: &shared_switches
-      clean_work:         True
-      do_refinediag:      False
-      do_analysis:        True
-
-experiments:
-  - name: "null_model_full"
-    pp:
-  - name: "null_model_0"
-    pp:
-  - name: "null_model_1"
-    pp:
-  - name: "null_model_2"
-    pp:

fre/pp/configure_script_yaml.py

@@ -122,6 +122,7 @@ def set_rose_suite(yamlfile: dict, rose_suite: metomi.rose.config.ConfigNode) ->
     """
     pp=yamlfile.get("postprocess")
     dirs=yamlfile.get("directories")
+    analysis=yamlfile.get("analysis")
 
     # set rose-suite items
     pa_scripts = ""
@@ -186,6 +187,34 @@ def set_rose_suite(yamlfile: dict, rose_suite: metomi.rose.config.ConfigNode) ->
         rose_suite.set( keys = ['template variables', 'DO_PREANALYSIS'],
                         value = 'False' )
 
+    # Set DO_ANALYSIS switch
+    # analysis_on is optional key for each component in the analysis yaml and 
+    # defaults to True if not specified.
+    # In the rose_suite.conf:
+    #  - if 'analysis_on: False' for all analysis components, set DO_ANALYSIS=False
+    #  - if 'analysis_on: True' for any analysis components, set DO_ANALYSIS=True
+    do_analysis_switch = []
+    if not analysis:
+        return
+
+    for an_key, an_value in analysis.items():
+        an_workflow_info = an_value["workflow"]
+        # if analysis_on key is actually set, evaluate and save its value in a list
+        if "analysis_on" in an_workflow_info:
+            do_analysis_switch.append(an_workflow_info["analysis_on"])
+        #if analysis_on key is NOT set, save its value as True in the list
+        else:
+            do_analysis_switch.append("True")
+
+    # if ANY of the analysis components do not set analysis_on or set analysis_on: True,
+    # set DO_ANALYSIS=True in the rose_suite.conf
+    if any(do_analysis_switch):
+        rose_suite.set( keys = ['template variables', 'DO_ANALYSIS'],
+                        value = 'True' )
+    else:
+        rose_suite.set( keys = ['template variables', 'DO_ANALYSIS'],
+                        value = 'False' )
+
     if dirs is not None:
         for key,value in dirs.items():
             rose_suite.set(keys=['template variables', key.upper()], value=quote_rose_values(value))

fre/pp/tests/AM5_example/am5.yaml

@@ -6,8 +6,8 @@ fre_properties:
   # amip
   - &EXP_AMIP_START     "19790101T0000Z"
   - &EXP_AMIP_END       "20200101T0000Z"
-  - &ANA_AMIP_START     "19800101T0000Z"
-  - &ANA_AMIP_END       "20200101T0000Z"
+  - &ANA_AMIP_START     "1980"
+  - &ANA_AMIP_END       "2020"
 
   - &PP_AMIP_CHUNK96    "P1Y"
   - &PP_AMIP_CHUNK384   "P1Y"

fre/pp/tests/AM5_example/yaml_include/pp.c96_amip.yaml

@@ -15,51 +15,44 @@ postprocess:
       xyInterp: *custom_interp
       interpMethod: "conserve_order2"
       inputRealm: 'atmos'
-      postprocess_on: False
     - type: "atmos"
       sources:
         - history_file: "atmos_month"
       sourceGrid: "cubedsphere"
       xyInterp: *PP_XYINTERP96
       interpMethod: "conserve_order2"
       inputRealm: 'atmos'
-      postprocess_on: True
     - type: "atmos_level_cmip"
       sources:
         - history_file: "atmos_level_cmip"
       sourceGrid: "cubedsphere"
       xyInterp: *PP_XYINTERP96
       interpMethod: "conserve_order2"
       inputRealm: 'atmos'
-      postprocess_on: False
     - type: "atmos_level"
       sources:
         - history_file: "atmos_month"
       sourceGrid: "cubedsphere"
       xyInterp: *PP_XYINTERP96
       interpMethod: "conserve_order2"
       inputRealm: 'atmos'
-      postprocess_on: False
     - type: "atmos_month_aer"
       sources:
         - history_file: "atmos_month_aer"
       sourceGrid: "cubedsphere"
       xyInterp: *PP_XYINTERP96
       interpMethod: "conserve_order1"
       inputRealm: 'atmos'
-      postprocess_on: False
     - type: "atmos_diurnal"
       sources:
         - history_file: "atmos_diurnal"
       sourceGrid: "cubedsphere"
       xyInterp: *PP_XYINTERP96
       interpMethod: "conserve_order2"
       inputRealm: 'atmos'
-      postprocess_on: False
     - type: "atmos_scalar"
       sources:
         - history_file: "atmos_scalar"
-      postprocess_on: True
     - type: "aerosol_cmip"
       xyInterp: *PP_XYINTERP96
       sources:

fre/pp/tests/AM5_example/yaml_include/settings.yaml

@@ -19,8 +19,6 @@ postprocess:
   switches: &shared_switches
     clean_work:              True
     do_atmos_plevel_masking: True
-    do_analysis:             True
-    do_analysis_only:        False
   preanalysis:
     vitals:
       script: "/path/to/vitals-script"

fre/pp/tests/test_configure_script_yaml.py

@@ -159,6 +159,31 @@ def test_cleanup():
     shutil.rmtree(f"{TEST_DIR}/configure_yaml_out")
     assert not Path(f"{TEST_DIR}/configure_yaml_out").exists()
 
+def test_rose_suite_DO_ANALYSIS():
+    """
+    """
+    rose_suite = metomi.rose.config.ConfigNode()
+    yaml_dict = {
+        "postprocess": {"settings": {"some_setting": "value"}},
+        "directories": {"pp_dir": "/some/path"},
+        "analysis": {
+                     "land-test": {
+                       "required": {
+                         "data_frequency": "mon",
+                         "date_range": ["19800101T0000Z", "20200101T0000Z"]
+                       },
+                       "workflow": {
+                         "components": ["land-test"],
+                         "script_type": "one-shot",
+                         "product": "ts",
+                         "chunk_size": "P1Y"
+                       }
+                     }
+                   }
+        }
+    csy.set_rose_suite(yaml_dict, rose_suite)
+    assert rose_suite.get(['template variables', 'DO_ANALYSIS']).value == 'True'
+
 ## to-do:
 # - mock wrong schema path
 # - any other raises missed

fre/yamltools/tests/AM5_example/am5.yaml

@@ -5,8 +5,8 @@ fre_properties:
   # amip
   - &EXP_AMIP_START     "1979"
   - &EXP_AMIP_END       "2020"
-  - &ANA_AMIP_START     1980
-  - &ANA_AMIP_END       2020
+  - &ANA_AMIP_START     "1980"
+  - &ANA_AMIP_END       "2020"
 
 #  - &PP_AMIP_CHUNK96    "P1Y"
   - &PP_AMIP_CHUNK384   "P1Y"

fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_compilefile.yaml

@@ -43,29 +43,6 @@ build:
   compileYaml: "compile.yaml" 
   platformYaml: "wrong_platforms.yaml"
 
-shared:
-  # directories shared across tools
-  directories: &shared_directories
-    history_dir:  !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
-    pp_dir:       !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
-    analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
-    ptmp_dir:     "/xtmp/$USER/ptmp"
-    fre_analysis_home: "/home/fms/local/opt/fre-analysis/test"
-
-  # shared pp settings
-  postprocess: 
-    settings: &shared_settings
-      history_segment:    "P1Y"
-      site:               "ppan"
-    switches: &shared_switches
-      do_statics:         True
-      do_timeavgs:        True
-      clean_work:         True
-      do_refinediag:      False
-      do_atmos_plevel_masking: True
-      do_preanalysis:     False
-      do_analysis:        True
-
 experiments:
   - name: "c96L65_am5f7b12r1_amip"
     pp:

fre/yamltools/tests/AM5_example/compile_yamls/compile_fail/am5-wrong_datatype.yaml

@@ -43,29 +43,6 @@ build:
   compileYaml: "wrong_compile.yaml"
   platformYaml: "wrong_platforms.yaml"
 
-shared:
-  # directories shared across tools
-  directories: &shared_directories
-    history_dir:  !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
-    pp_dir:       !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
-    analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
-    ptmp_dir:     "/xtmp/$USER/ptmp"
-    fre_analysis_home: "/home/fms/local/opt/fre-analysis/test"
-
-  # shared pp settings
-  postprocess: 
-    settings: &shared_settings
-      history_segment:    "P1Y"
-      site:               "ppan"
-    switches: &shared_switches
-      do_statics:         True
-      do_timeavgs:        True
-      clean_work:         True
-      do_refinediag:      False
-      do_atmos_plevel_masking: True
-      do_preanalysis:     False
-      do_analysis:        True
-
 experiments:
   - name: "c96L65_am5f7b12r1_amip"
     pp:

fre/yamltools/tests/AM5_example/settings.yaml

@@ -22,5 +22,3 @@ postprocess:
   switches:
     clean_work:         True
     do_atmos_plevel_masking: True
-    do_analysis:        True
-    do_analysis_only:   False