Merge pull request #25 from E3SM-Project/add-pcmdi

Add pcmdi

Author: forsyth2

.github/workflows/build_workflow.yml

@@ -54,6 +54,7 @@ jobs:
     needs: check-jobs-to-skip
     if: ${{ needs.check-jobs-to-skip.outputs.should_skip != 'true' }}
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     strategy:
       matrix:
         python-version: ["3.11", "3.12", "3.13"]
@@ -79,7 +80,7 @@ jobs:
           miniforge-variant: Miniforge3
           miniforge-version: latest
           environment-file: conda/dev.yml
-          channel-priority: flexible # Changed from strict to flexible
+          channel-priority: strict
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}
           channels: conda-forge
@@ -142,7 +143,7 @@ jobs:
   #         miniforge-variant: Miniforge3
   #         miniforge-version: latest
   #         environment-file: conda/dev.yml
-  #         channel-priority: flexible # Changed from strict to flexible
+  #         channel-priority: strict
   #         auto-update-conda: true
   #         python-version: "3.13" # Use stable Python version for docs
 

conda/dev.yml

@@ -18,10 +18,11 @@ dependencies:
   - beautifulsoup4
   - lxml
   - matplotlib-base >=3.8.2,<3.10
-  - mpas_tools >=0.21.0
+  - mpas_tools
   - netcdf4
   - numpy >=2.0,<3.0
   - output_viewer=1.3.3
+  - pcmdi_metrics>=3.9.3
   - xarray >=2023.02.0
   - xcdat >=0.7.3,<1.0
   # Testing
@@ -40,10 +41,13 @@ dependencies:
   - black ==25.1.0
   - flake8 ==7.3.0
   - isort ==6.0.1
-  - mypy ==1.18.2
+  #- mypy ==1.18.2
   - pre-commit ==4.3.0
   - types-PyYAML >=6.0.0
   # Developer Tools
   # =======================
   - tbump=6.9.0
   - ipykernel
+  # pip dependencies
+  - pip:
+    - mypy==1.18.2

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "matplotlib >=3.8.2,<3.10",
     "netcdf4",
     "numpy >=2.0,<3.0",
+    "pcmdi_metrics>=3.9.3",
     "xarray >=2023.02.0",
     "xcdat >=0.7.3,<1.0",
 ]
@@ -116,6 +117,11 @@ version = { attr = "zppy_interfaces.version.__version__" }
 # evolution of options.entry-points
 [project.scripts]
 zi-global-time-series = "zppy_interfaces.global_time_series.__main__:main"
+zi-pcmdi-link-observation = "zppy_interfaces.pcmdi_diags.link_observation:main"
+zi-pcmdi-mean-climate = "zppy_interfaces.pcmdi_diags.pcmdi_mean_cimate:main"
+zi-pcmdi-variability-modes = "zppy_interfaces.pcmdi_diags.pcmdi_variability_modes:main"
+zi-pcmdi-enso = "zppy_interfaces.pcmdi_diags.pcmdi_enso:main"
+zi-pcmdi-synthetic-plots = "zppy_interfaces.pcmdi_diags.pcmdi_synthetic_plots:main"
 
 [project.urls]
 Documentation = "https://docs.e3sm.org/zppy-interfaces"

tests/unit/pcmdi_diags/test_pcmdi_mean_climate.py

@@ -0,0 +1,27 @@
+from typing import List
+
+from zppy_interfaces.pcmdi_diags.pcmdi_mean_cimate import generate_mean_clim_cmds
+
+
+def test_generate_mean_clim_cmds():
+    # Sample of mean_climate default vars in zppy/defaults/default.ini
+    variables: List[str] = ["tauu", "tauv", "ta-200"]
+    # Example: /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_48/v3.LR.amip_0101/post/scripts/tmp.pcmdi_diags_mean_climate_model_vs_obs_2005-2014.915900.07Jq/pcmdi_diags/climo_ref_mean_climate_catalogue.json
+    obs_dic = {
+        "tauu": {"set": "default"},
+        "tauv": {"set": "default"},
+        "ta": {"set": "default"},  # Not in the example json file above
+    }
+    # Example: Appears after "AC" in .nc files in /lcrc/group/e3sm/ac.forsyth2/zppy_pr719_output/unique_id_48/v3.LR.amip_0101/post/scripts/tmp.pcmdi_diags_mean_climate_model_vs_obs_2005-2014.915900.07Jq/climo
+    case_id: str = "v20250923"
+    actual = generate_mean_clim_cmds(
+        variables=variables,
+        obs_dic=obs_dic,
+        case_id=case_id,
+    )
+    expected: List[str] = [
+        "mean_climate_driver.py -p parameterfile.py --vars tauu -r default --case_id v20250923",
+        "mean_climate_driver.py -p parameterfile.py --vars tauv -r default --case_id v20250923",
+        "mean_climate_driver.py -p parameterfile.py --vars ta-200 -r default --case_id v20250923",
+    ]
+    assert actual == expected

tests/unit/pcmdi_diags/test_pcmdi_setup.py

@@ -0,0 +1,23 @@
+from zppy_interfaces.pcmdi_diags.pcmdi_setup import (
+    DataCatalogueBuilder,
+    LandSeaMaskGenerator,
+)
+
+
+def test_DataCatalogueBuilder():
+    dcb = DataCatalogueBuilder("", [], "", [], [], "", "")
+
+    assert dcb._get_base_varname("ta-200") == "ta"
+    assert dcb._get_base_varname("ta_200") == "ta"
+    assert dcb._get_base_varname("pr") == "pr"
+
+
+def test_LandSeaMaskGenerator():
+    lsmg = LandSeaMaskGenerator("", "", "", "")
+    assert lsmg._parse_flag("True")
+    assert lsmg._parse_flag("Y")
+    assert lsmg._parse_flag("Yes")
+    assert lsmg._parse_flag("true")
+    assert lsmg._parse_flag("y")
+    assert lsmg._parse_flag("yes")
+    assert not lsmg._parse_flag("False")

tests/unit/pcmdi_diags/test_pcmdi_variability_modes.py

@@ -0,0 +1,66 @@
+from zppy_interfaces.pcmdi_diags.pcmdi_variability_modes import (
+    VariabilityMetricsCollector,
+    generate_varmode_cmds,
+)
+
+
+def test_VariabilityMetricsCollector():
+    vmc = VariabilityMetricsCollector(
+        ["mode"],
+        "png",
+        "mip",
+        "exp",
+        "model",
+        "relm",
+        "v20250923",
+        "dir_%(metric_type)",
+        "",
+    )
+    assert vmc.input_dir == "dir_variability_modes"
+    assert vmc.model_name == "mip.exp.model_relm"
+    assert vmc.seasons == ["DJF", "MAM", "JJA", "SON", "yearly", "monthly"]
+    assert vmc.fig_sets["MOV_eoftest"] == ["diagnostic_results", "EG_Spec*"]
+    assert vmc.fig_sets["MOV_compose"] == ["graphics", "*compare_obs"]
+    assert vmc.fig_sets["MOV_telecon"] == ["graphics", "*teleconnection"]
+    assert vmc.fig_sets["MOV_pattern"] == ["graphics", "*"]
+    assert (
+        vmc._classify_output_name("graphics", "mode", "DJF", "invalid.txt")
+        == "graphics_mode_DJF_unknown.png"
+    )
+    assert (
+        vmc._classify_output_name("graphics", "mode", "DJF", "North_test.txt")
+        == "graphics_mode_DJF_EG_Spec.png"
+    )
+    assert (
+        vmc._classify_output_name("graphics", "mode", "DJF", "_cbf_.txt")
+        == "graphics_mode_DJF_cbf.png"
+    )
+    assert (
+        vmc._classify_output_name("graphics", "mode", "DJF", "EOF1.txt")
+        == "graphics_mode_DJF_eof1.png"
+    )
+    assert (
+        vmc._classify_output_name("graphics", "mode", "DJF", "EOF2.txt")
+        == "graphics_mode_DJF_eof2.png"
+    )
+    assert (
+        vmc._classify_output_name("graphics", "mode", "DJF", "EOF3.txt")
+        == "graphics_mode_DJF_eof3.png"
+    )
+
+
+def test_generate_varmode_cmds():
+    actual = generate_varmode_cmds(
+        ["mode1", "mode2"],
+        "varOBS",
+        "reftyrs",
+        "reftyre",
+        "refname",
+        "refpath",
+        "v20250923",
+    )
+    expected = [
+        "variability_modes_driver.py -p parameterfile.py --variability_mode mode1 --eofn_mod 1 --eofn_obs 1 --varOBS varOBS --osyear reftyrs --oeyear reftyre --reference_data_name refname --reference_data_path refpath --case_id v20250923",
+        "variability_modes_driver.py -p parameterfile.py --variability_mode mode2 --eofn_mod 1 --eofn_obs 1 --varOBS varOBS --osyear reftyrs --oeyear reftyre --reference_data_name refname --reference_data_path refpath --case_id v20250923",
+    ]
+    assert actual == expected

tests/unit/pcmdi_diags/test_synthetic_plots_utils.py

@@ -0,0 +1,22 @@
+from zppy_interfaces.pcmdi_diags.synthetic_plots.utils import get_highlight_models
+
+
+def test_get_highlight_models():
+    all_models = [
+        "CESM2-FV2",
+        "CESM2-WACCM",
+        "CESM2-WACCM-FV2",
+        "GFDL-AM4",
+        "GFDL-CM4",
+        "GFDL-ESM4",
+        "E3SM-1-0",
+        "E3SM-2-0",
+    ]
+    model_name = ["CESM2-FV2", "E3SM"]
+    actual = get_highlight_models(all_models, model_name)
+    expected = [
+        "E3SM-1-0",
+        "E3SM-2-0",
+        "CESM2-FV2",
+    ]
+    assert actual == expected

tests/unit/pcmdi_diags/test_viewer.py

@@ -0,0 +1,6 @@
+from zppy_interfaces.pcmdi_diags.viewer import safe_join
+
+
+def test_safe_join():
+    assert safe_join("a", "b") == "a/b"
+    assert safe_join("a/", "b") == "a/b"