Add PCMDI Diags to zppy

Author: zhangshixuan1987

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 exclude: "docs|node_modules|migrations|.git|.tox"
-default_stages: [commit]
+default_stages: [pre-commit]
 fail_fast: true
 
 repos:

conda/chrisalis.yml

@@ -0,0 +1,34 @@
+name: zppy_dev
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # Base
+  # =================
+  - python=3.9.13
+  - pip=22.2.2
+  - configobj=5.0.6
+  - jinja2=3.1.2
+  - mache>=1.5.0
+  - mpas_tools>=0.15.0
+  - pillow=9.2.0
+  # Developer Tools
+  # =================
+  # If versions are updated, also update 'rev' in `.pre-commit-config.yaml`
+  - black=22.8.0        # version from https://anaconda.org/conda-forge/black
+  - flake8=5.0.4        # version from https://anaconda.org/conda-forge/flake8
+  # This line also implicitly installs isort
+  - flake8-isort=4.2.0  # version from https://anaconda.org/conda-forge/flake8-isort
+  - mypy=0.982          # version from https://anaconda.org/conda-forge/mypy
+  - pre-commit=2.20.0   # version from https://anaconda.org/conda-forge/pre-commit
+  - tbump=6.9.0
+  # Documentation
+  # If versions are updated, also update in `.github/workflows/build_workflow.yml`
+  # =================
+  - sphinx=5.2.3
+  - sphinx-multiversion=0.2.4
+  - sphinx_rtd_theme=1.0.0
+  # Need to pin docutils because 0.17 has a bug with unordered lists
+  # https://github.com/readthedocs/sphinx_rtd_theme/issues/1115
+  - docutils=0.16
+prefix: /home/ac.szhang/.conda/envs/zppy_dev

setup.py

@@ -18,7 +18,7 @@ def package_files(directory, prefixes, extensions):
 
 
 data_files = package_files(
-    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json"]
+    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json", "py"]
 )
 
 setup(

zppy/__main__.py

@@ -17,6 +17,7 @@
 from zppy.ilamb import ilamb
 from zppy.logger import _setup_custom_logger
 from zppy.mpas_analysis import mpas_analysis
+from zppy.pcmdi_diags import pcmdi_diags
 from zppy.tc_analysis import tc_analysis
 from zppy.ts import ts
 from zppy.utils import check_status, submit_script
@@ -247,6 +248,9 @@ def _launch_scripts(config: ConfigObj, script_dir, job_ids_file, plugins) -> Non
     # ilamb tasks
     existing_bundles = ilamb(config, script_dir, existing_bundles, job_ids_file)
 
+    # pcmdi_diags tasks
+    existing_bundles = pcmdi_diags(config, script_dir, existing_bundles, job_ids_file)
+
     # zppy external plugins
     for plugin in plugins:
         # Get plugin module function

zppy/defaults/default.ini

@@ -123,16 +123,21 @@ input_component = string(default="")
 
 [ts]
 area_nm = string(default="area")
+cmip_metadata = string(default="inclusions/e3sm_to_cmip/default_metadata.json")
+cmip_plevdata = string(default="inclusions/e3sm_to_cmip/vrt_remap_plev19.nc")
 # Days per file
 dpf = integer(default=30)
 extra_vars = string(default="")
+interp_vars = string(default="U,V,T,Q,RELHUM,OMEGA,Z3")
 # Time-steps per day
 tpd = integer(default=1)
 # Model component having generated input files (eam, eamxx, elm, mosart, ...)
 input_component = string(default="")
 
   [[__many__]]
   area_nm = string(default=None)
+  cmip_metadata = string(default=None)
+  cmip_plevdata = string(default=None)
   dpf = integer(default=None)
   extra_vars = string(default=None)
   tpd = integer(default=None)
@@ -150,6 +155,7 @@ input_component = string(default="")
   cmip_metadata = string(default=None)
   cmip_vars = string(default=None)
   input_component = string(default=None)
+  interp_vars = string(default=None)
 
 [tc_analysis]
 # NOTE: always overrides value in [default]
@@ -158,6 +164,229 @@ input_files = string(default="eam.h2")
 # If not set, zppy will attempt to infer this value from the topography file.
 res = string(default="")
 
+[pcmdi_diags]
+backend = string(default="mpl")
+cfg = string(default="")
+# Title of zppy-pcmdi diagnostics
+pcmdi_webtitle = string(default="E3SM-PMP Diagnostics")
+# Version of zppy-pcmdi code
+pcmdi_version = string(default="v3.8.2")
+# File of specified regions for mean climate calculation
+regions_specs = string(default="pcmdi_data/region/regions_specs.json")
+# File of observation data name for mean climate calculation
+reference_alias = string(default="pcmdi_data/reference/reference_alias.json")
+# Utility file with functions for zppy-pcmdi data processing
+pcmdi_zppy_util = string(default="pcmdi_data/utility/pcmdi_zppy_util.py")
+# Utility file with functions for zppy-pcmdi viewer page processing
+pcmdi_viewer_util = string(default="pcmdi_data/utility/pcmdi_viewer_util.py")
+# Template files for zppy-pcmdi viewer page processing
+pcmdi_viewer_template = string(default="pcmdi_data/viewer")
+# File of variable list to generate synthetic metrics plot
+synthetic_metrics = string(default="pcmdi_data/synthetic_metrics/synthetic_metrics_list.json")
+# Path to observation time-series data
+# Required for "mean_climate","variability_mode","enso"
+obs_ts = string(default="")
+# observational data sets (see reference_alias.json)
+# observation data tag in reference_alias
+obs_sets = string(default="default")
+# options specific for constructing pcmdi preferred file name conventions
+# required for "model_vs_obs" comparison
+model_name = string(default="e3sm.historical.v3-LR.0051")
+model_tableID = string(default="Amon")
+# required for "model_vs_model" comparison
+model_name_ref = string(default="ERA5")
+model_tableID_ref = string(default="Amon")
+# variables in the cmip6 table that can be potentially used by pcmdi
+# this list depends on the definition of cmip variable
+# required for "mean climate" diagnostics
+cmip_vars = string(default="pr,prw,psl,rlds,rldscs,rlut,rlutcs,rsut,rsutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rtmt,sfcWind,tas,tauu,tauv,ts,ta,ua,va,zg")
+# flag to process the land/sea mask within pcmdi
+generate_sftlf = string(default=True)
+# variables to be used by the pcmdi diagnostics
+# needs to setup for each subsections, defalut setup is the mean climate metrics
+vars = string(default="pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500")
+# sets of diagnostics from pcmdi package
+sets = string_list(default=list("mean_climate","variability_modes_atm","variability_modes_cpl","enso","synthetic_plots"))
+# options to identify subset of pcmdi drivers ("mean_climate","variability_mode","enso")
+subsets = string_list(default=list("mean_climate","variability_modes","enso"))
+###########################################################################################
+# Required for run_type="model_vs_model" runs, different from e3sm_diag,
+# model_vs_model in pcmdi referred to the comparision of two model simulations
+# with observations and cmip models.
+###########################################################################################
+#path for reference model data (time series)
+reference_data_path = string(default="")
+reference_data_path_ts = string(default="")
+# pcmdi_diags.py will set to match `years` if not specified
+ref_years = string_list(default=list(""))
+# The years increment for reference data
+ts_num_years_ref = integer(default=5)
+# Final year (i.e., the last available year) for the reference data
+ref_final_yr = string(default="")
+# Start year for the reference data
+ref_start_yr = string(default="")
+# reference model name
+ref_name = string(default="")
+# The years range for test model data
+ts_years = string_list(default=list(""))
+# The years increment for test model data
+ts_num_years = integer(default=5)
+# Set to true to swap test and ref when run_type="model_vs_model"
+swap_test_ref = boolean(default=False)
+##########################################################################################
+# below followed the setup in e3sm_diag but used for PCMDI workflow
+##########################################################################################
+# See url<need to work on document later>
+multiprocessing = boolean(default=True)
+# See url<need to work on document later>
+num_workers = integer(default=24)
+# See url<need to work on document later>
+figure_format = string(default="png")
+# comparision type (same as e3sm_diag)
+run_type = string(default="model_vs_obs")
+# Used to label the results directory
+# Options are "model_vs_obs" and "model_vs_model"
+tag = string(default="model_vs_obs")
+##########################################################################################
+# options for pcmdi mode varibility diagnostics
+# vars = "psl" for atm_modes
+# vars = "ts" for cpl_modes
+#########################################################################################
+#name of atmospheric modes varibility
+atm_modes = string(default="NAM,NAO,PNA,NPO,SAM,PSA1,PSA2")
+#name of coupled modes varibility
+cpl_modes = string(default="PDO,NPGO,AMO")
+#keywards for unit conversion in pcmdi (model)
+ModUnitsAdjust = string(default="")
+#keywards for unit conversion in pcmdi (observation)
+ObsUnitsAdjust = string(default="")
+#frequency of the model data
+frequency = string(default="mo")
+#options specific for mode varibility metrics in pcmdi
+seasons = string(default="monthly")
+landmask = string(default=False)
+RmDomainMean = string(default=True)
+EofScaling = string(default=False)
+ConvEOF = string(default=True)
+CBF = string(default=True)
+mov_plot_obs = string(default=True)
+mov_plot_model = string(default=True)
+mov_nc_out_obs = string(default=True)
+mov_nc_out_model = string(default=True)
+##########################################################################################
+# options for pcmdi enso diagnostics
+# vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt"
+##########################################################################################
+enso_groups = string(default="ENSO_perf,ENSO_proc,ENSO_tel")
+##########################################################################################
+# optional for mean climate diagnostics
+# vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,
+#         rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,
+#         va-850,zg-500"
+##########################################################################################
+# model data grid after remapping
+grid = string(default="180x360_aave")
+#default regions for mean climate metrics data
+#more options can be found at "regions_specs"
+#regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS,NHEX_ocean,SHEX_ocean,NHEX_land,SHEX_land,ocean_50S50N")
+regions = string(default="global,ocean,land,NHEX,SHEX,TROPICS")
+# save derived climatology data
+save_test_clims = string(default=True)
+# Regridding by pcmdi (required for mean climate)
+# OPTIONS: '1x1' for e3sm, '2.5x2.5' for cmip, or an actual cdms2 grid object,
+target_grid = string(default="1x1")
+# OPTIONS: "1px1p" for e3sm, "2p5x2p5" for cmip, description on the selected grid
+target_grid_string = string(default="1px1p")
+# OPTIONS: 'regrid2','esmf'
+regrid_tool = string(default="esmf")
+# OPTIONS: 'linear','conservative', only if tool is esmf
+regrid_method = string(default="regrid2")
+# OPTIONS: 'linear','conservative', only if tool is esmf
+regrid_method_ocn = string(default="conservative")
+###################################################################
+#options for synthetic plots with exisiting pcmdi cmip model metrics
+###################################################################
+synthetic_sets = string(default="portrait,parcoord")
+# path to pcmdi generated cmip metrics data
+cmip_enso_dir = string(default="")
+cmip_clim_dir = string(default="")
+cmip_movs_dir = string(default="")
+#group of pcmdi generated cmip metrics data (mip.exp.version)
+cmip_clim_set = string(default="cmip6.historical.v20250108")
+cmip_movs_set = string(default="cmip6.historical.v20220825")
+cmip_enso_set = string(default="cmip6.historical.v20210620")
+# options shared by pcmdi
+pcmdi_debug = string(default=False)
+
+  [[__many__]]
+  backend = string(default=None)
+  cfg = string(default=None)
+  vars = string(default=None)
+  grid = string(default=None)
+  cmip_enso_set = string(default=None)
+  cmip_clim__set = string(default=None)
+  cmip_movs_set = string(default=None)
+  cmip_enso_dir = string(default=None)
+  cmip_clim_dir = string(default=None)
+  cmip_movs_dir = string(default=None)
+  synthetic_sets = string(default=None)
+  reference_alias = string(default=None)
+  regions_specs = string(default=None)
+  pcmdi_webtitle = string(default=None)
+  pcmdi_version = string(default=None)
+  pcmdi_zppy_util = string(default=None)
+  pcmdi_viewer_util = string(default=None)
+  pcmdi_viewer_template = string(default=None)
+  synthetic_metrics = string(default=None)
+  multiprocessing = boolean(default=None)
+  num_workers = integer(default=None)
+  obs_ts = string(default=None)
+  figure_format = string(default=None)
+  ref_name = string(default=None)
+  ref_years = string_list(default=None)
+  ref_start_yr = string(default=None)
+  ref_final_yr = string(default=None)
+  reference_data_path = string(default=None)
+  reference_data_path_ts = string(default=None)
+  run_type = string(default=None)
+  sets = string_list(default=None)
+  swap_test_ref = boolean(default=None)
+  tag = string(default=None)
+  ts_years = string_list(default=None)
+  ts_num_years = integer(default=None)
+  ts_num_years_ref = integer(default=None)
+  target_grid = string(default=None)
+  target_grid_string = string(default=None)
+  regrid_tool = string(default=None)
+  regrid_method = string(default=None)
+  regrid_method_ocn = string(default=None)
+  obs_sets = string(default=None)
+  regions = string(default=None)
+  save_test_clims = string(default=None)
+  seasons = string(default=None)
+  RmDomainMean = string(default=None)
+  EofScaling = string(default=None)
+  ConvEOF = string(default=None)
+  CBF = string(default=None)
+  subsets = string_list(default=None)
+  landmask = string(default=None)
+  frequency = string(default=None)
+  generate_sftlf = string(default=None)
+  atm_modes = string(default=None)
+  cpl_modes = string(default=None)
+  enso_groups = string(default=None)
+  ModUnitsAdjust = string(default=None)
+  ObsUnitsAdjust = string(default=None)
+  model_name = string(default=None)
+  model_name_ref = string(default=None)
+  model_tableID = string(default=None)
+  model_vars = string(default=None)
+  mov_nc_out_obs = string(default=None)
+  mov_nc_out_model = string(default=None)
+  mov_plot_obs = string(default=None)
+  mov_plot_model = string(default=None)
+  pcmdi_debug = string(default=None)
+
 [e3sm_diags]
 # See https://e3sm-project.github.io/e3sm_diags/_build/html/master/available-parameters.html
 backend = string(default="mpl")