Merge branch 'main' into pip_refactor

wrongkindofdoctor · wrongkindofdoctor · commit 5bce78f662b4 · 2025-04-30T17:26:26.000-04:00
diff --git a/MDTF_diagnostics/data/fieldlist_CMIP.jsonc b/MDTF_diagnostics/data/fieldlist_CMIP.jsonc
@@ -248,6 +248,7 @@
     },
     "tos": {
       "standard_name": "sea_surface_temperature",
+      "realm": "ocean",
       "units": "degC",
       "ndim": 3
     },
@@ -258,11 +259,6 @@
       "modifier": "atmos_height",
       "ndim": 3
     },
-    "tos": {
-      "standard_name": "sea_surface_temperature",
-      "units": "K",
-      "ndim": 3
-    },
     // radiative fluxes:
     "rsus": {
       "standard_name": "surface_upwelling_shortwave_flux_in_air",
diff --git a/MDTF_diagnostics/diagnostics/mar/SST_bias_NOAA_OISSTv2/SST_bias_NOAA_OISSTv2.ipynb b/MDTF_diagnostics/diagnostics/mar/SST_bias_NOAA_OISSTv2/SST_bias_NOAA_OISSTv2.ipynb
@@ -19,7 +19,7 @@
     "\n",
     "\n",
     "# For testing and development purposes, enter a start year, end year, and\n",
-    "# an dora id number to analyze. The value of dora_id can also be a direct\n",
+    "# a dora id number to analyze. The value of dora_id can also be a direct\n",
     "# path to a /pp directory.\n",
     "\n",
     "config = {\n",
@@ -64,12 +64,10 @@
    ]
   },
   {
-   "cell_type": "code",
-   "id": "dfcdc828-ad45-4ef0-92bc-05df9c036270",
    "metadata": {},
+   "cell_type": "code",
    "source": [
     "import doralite\n",
-    "import glob\n",
     "import momlevel\n",
     "import subprocess\n",
     "import os\n",
@@ -85,6 +83,7 @@
     "\n",
     "from matplotlib.colors import ListedColormap, BoundaryNorm"
    ],
+   "id": "c57e0f8b0c23ad83",
    "outputs": [],
    "execution_count": null
   },
@@ -149,11 +148,7 @@
   {
    "cell_type": "code",
    "id": "382835f3-a3d7-419f-a92f-e65808230c59",
-   "metadata": {
-    "jupyter": {
-     "is_executing": true
-    }
-   },
+   "metadata": {},
    "source": [
     "# Determine what files are needed (leave this up to the developer for flexibility)\n",
     "# This is an example of what someone might do:\n",
diff --git a/MDTF_diagnostics/diagnostics/mar/SST_bias_NOAA_OISSTv2/settings.jsonc b/MDTF_diagnostics/diagnostics/mar/SST_bias_NOAA_OISSTv2/settings.jsonc
@@ -4,7 +4,7 @@
     "long_name" : "Sea Surface Temperature Bias - NOAA OISSTv2",
     "convention": "gfdl",
     "mar": true,
-    "description" : "This notebook plots the SST bias of a model simulation vs. climataology derived from NOAA OISSTv2",
+    "description" : "This notebook plots the SST bias of a model simulation vs. climatology derived from NOAA OISSTv2",
     "runtime_requirements": {
         "python3": ["matplotlib", "xarray", "netCDF4"]
     }
diff --git a/MDTF_diagnostics/src/conda/env_dev.yml b/MDTF_diagnostics/src/conda/env_dev.yml
@@ -10,9 +10,9 @@ dependencies:
 - netCDF4=1.6.5
 - cftime=1.6.2
 - scikit-learn=1.4.2
-- xesmf=0.8.1
-- esmf=8.4.2
-- esmpy=8.4.2
+- xesmf=0.8.9
+- esmf=8.8.1
+- esmpy=8.8.1
 - xarray=2025.1.2
 - matplotlib=3.8.2
 - cartopy=0.22.0
diff --git a/MDTF_diagnostics/src/conda/env_python3_base.yml b/MDTF_diagnostics/src/conda/env_python3_base.yml
@@ -20,10 +20,11 @@ dependencies:
 - dask=2025.2.0
 - numba=0.60.0
 - scikit-learn=1.4.2
-- xesmf=0.8.7
-- esmf=8.4.2
-- esmpy=8.4.2
+- xesmf=0.8.9
+- esmf=8.8.1
+- esmpy=8.8.1
 - gsw=3.6.17
+- cfunits=3.3.6
 - metpy=1.5.1
 #- enso_metrics=3.1.1 Required by pcmdi_metrics, but no support for python>3.10
 #- pcmdi_metrics=3.0.2 Not used by current PODs-implement later
diff --git a/MDTF_diagnostics/src/conda/env_python3_base_micromamba.yml b/MDTF_diagnostics/src/conda/env_python3_base_micromamba.yml
@@ -20,9 +20,9 @@ dependencies:
 - dask=2025.2.0
 - numba=0.60.0
 - scikit-learn=1.4.2
-- xesmf=0.8.1
-- esmf=8.4.2
-- esmpy=8.4.2
+- xesmf=0.8.9
+- esmf=8.8.1
+- esmpy=8.8.1
 - gsw=3.6.17
 - metpy=1.5.1
 #- enso_metrics=3.1.1 Required by pcmdi_metrics, but no support for python>3.10
diff --git a/MDTF_diagnostics/tools/run-MDTF/config/pod_config.json b/MDTF_diagnostics/tools/run-MDTF/config/pod_config.json
@@ -2,21 +2,26 @@
   "Wheeler_Kiladis": {
     "realm": "atmos_cmip",
     "frequency": "day",
-    "vars": ["rlut","pr","wap500","ua200","ua850"]
+    "vars": ["rlut", "pr", "wap500", "ua200", "ua850"]
   },
   "MJO_suite": {
     "realm": "atmos_cmip",
     "frequency": "day",
-    "vars": ["rlut","pr","ua200","ua850","va200","va850"]
+    "vars": ["rlut", "pr", "ua200", "ua850", "va200", "va850"]
   },
   "ocn_surf_flux_diag": {
     "realm": "atmos_cmip",
     "frequency": "day",
-    "vars": ["ts","psl","sfcWind","huss","hfls","pr"]
+    "vars": ["ts", "psl", "sfcWind", "huss", "hfls" ,"pr"]
   },
   "forcing_feedback": {
     "realm": "atmos_cmip",
     "frequency": "mon",
-    "vars": ["ts","ta","hus","rsus","rsuscs","rsds","rsdscs","rsdt","rsut","rsutcs","rlut","rlutcs"]
-  }
+    "vars": ["ts", "ta", "hus", "rsus", "rsuscs", "rsds", "rsdscs", "rsdt", "rsut", "rsutcs", "rlut", "rlutcs"]
+  },
+  "SST_bias_NOAA_OISSTv2": {
+    "realm": "ocean_monthly",
+    "frequency": "mon",
+    "vars": ["tos", "areacello"]
+  }  
 }
diff --git a/MDTF_diagnostics/tools/run-MDTF/doc/README.md b/MDTF_diagnostics/tools/run-MDTF/doc/README.md
@@ -1,5 +1,9 @@
 # run-MDTF
 Shell driver script to run the MDTF-diagnostics package with the GFDL FRE workflow <br />
+This script automatially generates an intake-ESM catalog with files in the target directory
+using the GFDL catalogBuilder tool with --slow option to ensure that all required variable
+metadata are populated. If a catalog exists in the input data directory or the output directory,
+ the script will skip the catalog generation step.
 
 Command-line usage:
 
@@ -15,3 +19,14 @@ Input:
 - `-o`: path to directory to write the output
 - `-s`: start year of the dataset
 - `-e`: end year of the dataset
+- `-c`: (optional) data convention, cmip (default), gfdl, or cesm
+
+Example using GFDL central MDTF-diagnostics installation. Note that the resulting catalog will not include static files with
+variables required by the example mar SST_bias_NOAA_OISSTvs notebook. In the following example, the information for `ocean_monthly.static.nc`
+needs to be entered manually at this time, so the user needs to call `run-MDTF.sh` twice:
+
+```
+$ /home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics/tools/run-MDTF/run-MDTF.sh
+-i /archive/John.Krasting/fre_om5/FMS2024.02_om5_20250206/om5_b08/gfdl.ncrc5-intel23-prod/pp/ocean_monthly/ts
+-o ~/mdtf_test_out -s 1978 -e 1988 -l config/pod_config.json
+```
diff --git a/MDTF_diagnostics/tools/run-MDTF/run-MDTF.sh b/MDTF_diagnostics/tools/run-MDTF/run-MDTF.sh
@@ -5,8 +5,8 @@
 
 # dir references
 run_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+echo "${run_dir}"
 mdtf_dir=/home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics
-#mdtf_dir=/home/Jacob.Mims/mdtf/MDTF-diagnostics
 activate=/home/oar.gfdl.mdtf/miniconda3/bin/activate
 
 #TEST: /archive/jpk/fre/FMS2024.02_OM5_20240819/CM4.5v01_om5b06_piC_noBLING_xrefine_test4/gfdl.ncrc5-intel23-prod-openmp/pp/ starts 0001
@@ -23,7 +23,7 @@ usage() {
 tempdir=""
 pod_config=""
 declare -a pods=()
-while getopts "hi:o:s:e:p:t:l:" arg; do
+while getopts "hi:o:s:e:p:t:l:c:" arg; do
    case "${arg}" in
       h) 
          usage
@@ -60,76 +60,87 @@ while getopts "hi:o:s:e:p:t:l:" arg; do
          ;;
       l)
          pod_config="${OPTARG}"
+         ;;
+      c)
+         convention="${OPTARG}"
    esac
 done
+
 shift $((OPTIND-1))
-if ! [ -d $outdir/config ]; then
-   mkdir -p $outdir/config
+if ! [ -d "${outdir}"/config ]; then
+   mkdir -p "${outdir}"/config
 fi
-if [ -z $tempdir ]; then
-   wkdir=$outdir
+
+if [ -z "${tempdir}" ]; then
+   wkdir="${outdir}"
 else
-   wkdir=$tempdir
+   wkdir="${tempdir}"
 fi
-if [ -z $pod_config ]; then 
-   pod_config="$run_dir/config/pod_config.json"
+
+if [ -z "${pod_config}" ]; then
+   pod_config="${run_dir}/config/pod_config.json"
 fi
 
 # check to see if catalog exists
 #  ^..^
 # /o  o\   
 # oo--oo~~~
-echo "looking for catalog in $ppdir"
-cat=$(grep -s -H "esmcat_version" $ppdir/*.json  | cut -d: -f1)
-if [[ "$cat" == "" ]]; then
-   env=/nbhome/fms/conda/envs/fre-2025.01
-   source $activate $env
-   fre catalog builder --overwrite $ppdir $outdir/catalog 
-   cat=$outdir/catalog.json
-   echo "new catalog generated: $cat"
-else
-   echo "found catalog: $cat"
-fi
+output_catalog="${outdir}"/catalog.json
+if [ ! -f "${output_catalog}" ]; then
+      echo "Looking for catalog in ${ppdir}"
+      input_catalog=$(grep -s -H "esmcat_version" "${ppdir}"/*.json  | cut -d: -f1)
+   if [[ "${input_catalog}" == "" ]]; then
+      env=/nbhome/fms/conda/envs/fre-2025.01
+      source $activate ${env}
+      fre catalog builder --slow --overwrite "${ppdir}" "${outdir}"/catalog
+      echo "New catalog generated: ${output_catalog}"
+   else
+      echo "found catalog: ${input_catalog}"
+   fi
 
+fi
 # edit template config file
-cp $run_dir/config/template_config.jsonc $outdir
-f=$outdir/template_config.jsonc
-if [ ! -f $f ]; then
-   echo "ERROR: failed to find $f"
+cp "${run_dir}"/config/template_config.jsonc "${outdir}"
+f=${outdir}/template_config.jsonc
+if [ ! -f "${f}" ]; then
+   echo "ERROR: failed to find ${f}"
    exit 0
 fi
 config='"DATA_CATALOG": "",'
-config_edit='"DATA_CATALOG": "'"${cat}"'",'
-sed -i "s|$config|$config_edit|ig" $f
+config_edit='"DATA_CATALOG": "'"${output_catalog}"'",'
+sed -i "s|${config}|${config_edit}|ig" "${f}"
 config='"WORK_DIR": "",'
 config_edit='"WORK_DIR": "'"${wkdir}"'",'
-sed -i "s|$config|$config_edit|ig" $f
+sed -i "s|${config}|${config_edit}|ig" "${f}"
 config='"OUTPUT_DIR": "",'
 config_edit='"OUTPUT_DIR": "'"${outdir}"'",'
-sed -i "s|$config|$config_edit|ig" $f
+sed -i "s|${config}|${config_edit}|ig" "${f}"
 config='"startdate": "",'
 config_edit='"startdate": "'"${startyr}"'",'
-sed -i "s|$config|$config_edit|ig" $f
+sed -i "s|${config}|${config_edit}|ig" "${f}"
 config='"enddate": ""'
 config_edit='"enddate": "'"${endyr}"'"'
-sed -i "s|$config|$config_edit|ig" $f
-echo "edited file $f"
+sed -i "s|${config}|${config_edit}|ig" "${f}"
+config_edit='"convention": "'"${convention}"'"'
+sed -i "s|${config}|${config_edit}|ig" "${f}"
+echo "edited file ${f}"
 
 # load mdtf base conda env
 env=/home/oar.gfdl.mdtf/miniconda3/envs/_MDTF_base
-source $activate $env
+source ${activate} ${env}
 #generate config files
-python $run_dir/scripts/gen_config.py $outdir/ $pod_config
+python "${run_dir}"/scripts/gen_config.py "${outdir}"/ "${pod_config}"
 
 # launch the mdtf with the config files
-for f in $(ls ${outdir}/config) ; do
-   echo "launching MDTF with $f"
-   "$mdtf_dir"/mdtf -f $outdir/config/$f
+file_list=$( ls "${outdir}"/config )
+for f in ${file_list}; do
+   echo "Launching MDTF-diagnostics with configuration ${f}"
+   "${mdtf_dir}"/mdtf -f "${outdir}/config/${f}"
 done
 
 # consolidate outputs into index
-cp $run_dir/scripts/index.html $outdir/
-cp $run_dir/scripts/mdtf_diag_banner.png $outdir/
-python $run_dir/scripts/gen_index.py $outdir/ $pod_config
+cp "${run_dir}"/scripts/index.html "${outdir}"/
+cp "${run_dir}"/scripts/mdtf_diag_banner.png "${outdir}"/
+python "${run_dir}"/scripts/gen_index.py "${outdir}"/ "${pod_config}"
 
 exit 0
diff --git a/MDTF_diagnostics/tools/run-MDTF/run-MDTF_frepp.csh b/MDTF_diagnostics/tools/run-MDTF/run-MDTF_frepp.csh
@@ -36,5 +36,5 @@ endif
 
 set ppdir = `echo ${in_data_dir} | sed 's|\(.*pp\).*|\1|'`
 
-/home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics/tools/run-MDTF/run-MDTF.sh -i ${ppdir} -o ${out_dir}/mdtf -s $yr1 -e $yr2
+/home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics/tools/run-MDTF/run-MDTF.sh -i ${ppdir} -o ${out_dir}/mdtf -s ${yr1} -e ${yr2}
 
diff --git a/MDTF_diagnostics/tools/run-MDTF/scripts/gen_config.py b/MDTF_diagnostics/tools/run-MDTF/scripts/gen_config.py
@@ -8,7 +8,7 @@
     pods = json.load(f)
 
 # load template config information
-with open(sys.argv[1]+'template_config.jsonc') as f:
+with open(sys.argv[1] + 'template_config.jsonc') as f:
     template_config = json.load(f)    
 
 # create dict object for each pod
@@ -20,12 +20,12 @@
     config_files[config_name]['case_list']['case_name']['realm'] = config_realm
     config_files[config_name]['case_list'][config_realm] = config_files[config_name]['case_list'].pop('case_name')
 
-# add pods to cooresponding config file
+# add pods to corresponding config file
 for p in pods:
     config_file = f"{p}_config"    
     config_files[config_file]['pod_list'].append(p) 
     
 #write out config files
 for c in config_files:
-    with open(sys.argv[1]+f"config/{c}.jsonc", "w") as f:
+    with open(sys.argv[1] + f"config/{c}.jsonc", "w") as f:
         f.write(json.dumps(config_files[c], indent=2))     
diff --git a/doc/sphinx/dev_gfdl.rst b/doc/sphinx/dev_gfdl.rst
@@ -0,0 +1,52 @@
+.. _ref-dev-gfdl:
+
+Maintaining the GFDL MDTF-diagnostics installation
+==================================================
+
+Updating the main branch of the oar.gfdl.mdtf MDTF-diagnostics repo
+-------------------------------------------------------------------
+
+A copy of the NOAA-GFDL/MDTF-diagnostics package is located in /home/oar.gfdl.mdtf/mdtf
+and is maintained by developers with oar.gfdl.mdtf role account access. The
+central installation is synced with main branch at this time. To update the main branch,
+log into the role account, `cd` to the /home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics, and run `git pull`.
+
+Maintaining miniconda3 environments
+-----------------------------------
+
+A central installation of miniconda3 with the MDTF-diagnostics _MDTF_base, _MDTF_python3_base, _MDTF_NCL_base,
+_MDTF_R_base, and _MDTF_dev environments located in /home/oar.gfdl.mdtf/miniconda3. To access conda binaries directly,
+open a bash shell and run `source ~/.bashrc`. The conda environments need to be updated
+when the environment files in MDTF-diagnostics/src/conda are changed on the main branch. After logging into the
+oar.gfdl.mdtf role account, `cd` to /home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics,
+pull in changes to the MDTF-diagnostics repo, and run `/src/conda/conda_env_setup.sh -e [environment name] --
+conda_root /home/oar.gfdl.mdtf/miniconda3 --env_dir /home/oar.gfdl.mdtf/miniconda3/envs`.
+`[environment name]` is "base" for the _MDTF_base environment, "dev" for the _MDTF_dev environment,
+"python3_base" for the _MDTF_python3_base environment, "NCL_base" for the _MDTF_NCL_base environment, and "R_base",
+for the _MDTF_R_base environment.
+
+Exising environments may require complete removal before updating, as continuously overwriting sometimes
+fails to update all packages. You can verify package versions in a particular environment match the specs in the
+corresponding environment file by running `conda activate _MDTF_[environment name]`, followed by `conda list`.
+To delete a conda environment, run `conda remove -n ENV_NAME --all`. Reinstall the environment using
+the `conda_env_setup.sh` call in the previous instructions.
+
+If miniconda3 itself requires updating, completely delete it by running `rm -rf /home/oar.gfdl.mdtf/miniconda3`,
+then remove the configuration files with `rm -rf ~/.condarc ~/.conda ~/.continuum`.
+
+Download a tarball of the latest miniconda3 version using `wget`, `curl`, etc..., and install it in
+the /home/oar.gfdl.mdtf directory using the `links and instructions
+<https://www.anaconda.com/docs/getting-started/miniconda/main>`__ on the Conda website. Reinstall the MDTF-diagnostics
+environments by running `/src/conda/conda_env_setup.sh --all --conda_root
+/home/oar.gfdl.mdtf/miniconda3 --env_dir /home/oar.gfdl.mdtf/miniconda3/envs`. Note that the _MDTF_dev environment
+needs to be installed with `-e dev`, as `--all` only installs the base environments.
+
+Adding copies of POD observational data to the oar.gfdl.mdtf/inputdata directory
+--------------------------------------------------------------------------------
+
+Copies of POD bservational datasets are located the /home/oar.gfdl.mdtf/mdtf/inputdata directory.
+The data are available in the NCAR MDTF obs_data collection on `Globus 
+<https://app.globus.org/file-manager?origin_id=87726236-cbdd-4a91-a904-7cc1c47f8912&origin_path=%2F&two_pane=false>`__.
+In addition, GFDL developers who are responsible for reviewing POD pull requests can ask POD developers to email
+tarballs with observational and test datasets. This may be necessary to debug issues with the framework that POD
+developers encounter when integrating their code into the package.
diff --git a/doc/sphinx/pod_dev_toc.rst b/doc/sphinx/pod_dev_toc.rst

Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@`
`4`	`4`	`"long_name" : "Sea Surface Temperature Bias - NOAA OISSTv2",`
`5`	`5`	`"convention": "gfdl",`
`6`	`6`	`"mar": true,`
`7`		`- "description" : "This notebook plots the SST bias of a model simulation vs. climataology derived from NOAA OISSTv2",`
	`7`	`+ "description" : "This notebook plots the SST bias of a model simulation vs. climatology derived from NOAA OISSTv2",`
`8`	`8`	`"runtime_requirements": {`
`9`	`9`	`"python3": ["matplotlib", "xarray", "netCDF4"]`
`10`	`10`	`}`
Original file line number	Diff line number	Diff line change
`@@ -36,5 +36,5 @@ endif`
`36`	`36`
`37`	`37`	set ppdir = `echo ${in_data_dir} \| sed 's\|\(.pp\).\|\1\|'`
`38`	`38`
`39`		`-/home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics/tools/run-MDTF/run-MDTF.sh -i ${ppdir} -o ${out_dir}/mdtf -s $yr1 -e $yr2`
	`39`	`+/home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics/tools/run-MDTF/run-MDTF.sh -i ${ppdir} -o ${out_dir}/mdtf -s ${yr1} -e ${yr2}`
`40`	`40`