NOAA-GFDL · wrongkindofdoctor · Apr 11, 2025 · Apr 11, 2025 · Apr 11, 2025 · Apr 11, 2025
@@ -0,0 +1,8 @@
+# run-MDTF
+MDTF shell driver to assist model development at the GFDL <br />
+Command-line usage:
+```
+run-mdtf.sh -i /path/to/pp/dir/pp -o out_dir/mdtf -s startyr -e endyr
+```
+This script requires a user inputted `pod_config.json`. One is supplied by default, but may need to be updated in order to launch more PODs or update realm information!
+A new one can be supplied with the `-l` OPTARG.
@@ -0,0 +1,22 @@
+{
+  "Wheeler_Kiladis": {
+    "realm": "atmos_cmip",
+    "frequency": "day",
+    "vars": ["rlut","pr","wap500","ua200","ua850"]
+  },
+  "MJO_suite": {
+    "realm": "atmos_cmip",
+    "frequency": "day",
+    "vars": ["rlut","pr","ua200","ua850","va200","va850"]
+  },
+  "ocn_surf_flux_diag": {
+    "realm": "atmos_cmip",
+    "frequency": "day",
+    "vars": ["ts","psl","sfcWind","huss","hfls","pr"]
+  },
+  "forcing_feedback": {
+    "realm": "atmos_cmip",
+    "frequency": "mon",
+    "vars": ["ts","ta","hus","rsus","rsuscs","rsds","rsdscs","rsdt","rsut","rsutcs","rlut","rlutcs"]
+  }
+}
@@ -0,0 +1,45 @@
+{
+  "pod_list" : [],
+  "case_list":
+   {
+     "case_name":
+      {
+        "model": "test",
+        "convention": "CMIP",
+        "startdate": "",
+        "enddate": "",
+        "realm": ""
+      }
+   },
+  "DATA_CATALOG": "",
+
+  "OBS_DATA_ROOT": "/home/oar.gfdl.mdtf/mdtf/inputdata/obs_data",
+
+  "WORK_DIR": "",
+
+  "OUTPUT_DIR": "",
+
+  "conda_root": "/home/oar.gfdl.mdtf/miniconda3",
+
+  "conda_env_root": "/home/oar.gfdl.mdtf/miniconda3/envs",
+
+  "micromamba_exe": "",
+
+  "run_pp": true,
+  "translate_data": true,
+  "save_ps": false,
+
+  "large_file": false,
+
+  "save_pp_data": false,
+
+  "make_variab_tar": false,
+
+  "make_multicase_figure_html": false,
+
+  "overwrite": false,
+
+  "user_pp_scripts" : [],
+
+  "verbose": 1
+}
@@ -0,0 +1,135 @@
+#!/bin/bash -f
+#SBATCH --job-name=run-MDTF.sh
+#SBATCH --time=4:00:00
+#set -x
+
+# dir references
+run_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+mdtf_dir=/home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics
+#mdtf_dir=/home/Jacob.Mims/mdtf/MDTF-diagnostics
+activate=/home/oar.gfdl.mdtf/miniconda3/bin/activate
+
+#TEST: /archive/jpk/fre/FMS2024.02_OM5_20240819/CM4.5v01_om5b06_piC_noBLING_xrefine_test4/gfdl.ncrc5-intel23-prod-openmp/pp/ starts 0001
+
+#TEST 2: /archive/djp/am5/am5f7b12r1/c96L65_am5f7b12r1_amip/gfdl.ncrc5-intel23-classic-prod-openmp/pp/ starts 1990
+
+usage() {
+   echo "USAGE: run-mdtf.sh -i /path/to/pp/dir/pp -o out_dir/mdtf -s startyr -e endyr"
+   echo "ADDITONAL OPTS:"
+   echo "-l: custom config file for pods (default: config/pod_config.json) this can be used to set which PODs you would like to run and define the realm to grab vars from"
+}
+
+# handle arguments
+tempdir=""
+pod_config=""
+declare -a pods=()
+while getopts "hi:o:s:e:p:t:l:" arg; do
+   case "${arg}" in
+      h) 
+         usage
+	 exit 
+         ;;
+      i) 
+         if [ -d $OPTARG ]; then
+            ppdir="${OPTARG}"
+         else
+            echo "ERROR: $1 is not a directory"
+            usage
+            exit
+         fi
+         ;;
+      o)
+         if [ -d "${OPTARG}" ]; then
+            outdir="${OPTARG}"
+         else
+            mkdir -p "${OPTARG}"
+            outdir="${OPTARG}"
+         fi
+         ;;      
+      s)
+         startyr="${OPTARG}"
+         ;;
+      e)
+         endyr="${OPTARG}"
+         ;;
+      p)
+         pods+=("$OPTARG")
+         ;;
+      t)
+         tempdir="${OPTARG}"
+         ;;
+      l)
+         pod_config="${OPTARG}"
+   esac
+done
+shift $((OPTIND-1))
+if ! [ -d $outdir/config ]; then
+   mkdir -p $outdir/config
+fi
+if [ -z $tempdir ]; then
+   wkdir=$outdir
+else
+   wkdir=$tempdir
+fi
+if [ -z $pod_config ]; then 
+   pod_config="$run_dir/config/pod_config.json"
+fi
+
+# check to see if catalog exists
+#  ^..^
+# /o  o\   
+# oo--oo~~~
+echo "looking for catalog in $ppdir"
+cat=$(grep -s -H "esmcat_version" $ppdir/*.json  | cut -d: -f1)
+if [[ "$cat" == "" ]]; then
+   env=/nbhome/fms/conda/envs/fre-2025.01
+   source $activate $env
+   fre catalog builder --overwrite $ppdir $outdir/catalog 
+   cat=$outdir/catalog.json
+   echo "new catalog generated: $cat"
+else
+   echo "found catalog: $cat"
+fi
+
+# edit template config file
+cp $run_dir/config/template_config.jsonc $outdir
+f=$outdir/template_config.jsonc
+if [ ! -f $f ]; then
+   echo "ERROR: failed to find $f"
+   exit 0
+fi
+config='"DATA_CATALOG": "",'
+config_edit='"DATA_CATALOG": "'"${cat}"'",'
+sed -i "s|$config|$config_edit|ig" $f
+config='"WORK_DIR": "",'
+config_edit='"WORK_DIR": "'"${wkdir}"'",'
+sed -i "s|$config|$config_edit|ig" $f
+config='"OUTPUT_DIR": "",'
+config_edit='"OUTPUT_DIR": "'"${outdir}"'",'
+sed -i "s|$config|$config_edit|ig" $f
+config='"startdate": "",'
+config_edit='"startdate": "'"${startyr}"'",'
+sed -i "s|$config|$config_edit|ig" $f
+config='"enddate": ""'
+config_edit='"enddate": "'"${endyr}"'"'
+sed -i "s|$config|$config_edit|ig" $f
+echo "edited file $f"
+
+# load mdtf base conda env
+env=/home/oar.gfdl.mdtf/miniconda3/envs/_MDTF_base
+source $activate $env
+#generate config files
+python $run_dir/scripts/gen_config.py $outdir/ $pod_config
+
+# launch the mdtf with the config files
+for f in $(ls ${outdir}/config) ; do
+   echo "launching MDTF with $f"
+   "$mdtf_dir"/mdtf -f $outdir/config/$f
+done
+
+# consolidate outputs into index
+cp $run_dir/scripts/index.html $outdir/
+cp $run_dir/scripts/mdtf_diag_banner.png $outdir/
+python $run_dir/scripts/gen_index.py $outdir/ $pod_config
+
+exit 0
@@ -0,0 +1,40 @@
+#!/bin/csh -f
+#--------------------------------------
+#PBS -N mdtf_frepp_driver
+#PBS -l size=1
+#PBS -l walltime=60:00:00
+#PBS -r y
+#PBS -j oe
+#PBS -o
+#PBS -q batch
+#--------------------------------------
+
+# clean up tmpdir
+wipetmp
+
+# fields set by frepp
+set argu
+set descriptor
+set in_data_dir
+set out_dir
+set WORKDIR
+set mode
+set yr1
+set yr2
+set databegyr
+set dataendyr
+set datachunk
+set staticfile
+set script_path
+set fremodule 
+set mode = "GFDL"
+
+if (-d ${out_dir}/mdtf) then
+  echo "Output directory already exists, removing"
+  rm -fr ${out_dir}/mdtf
+endif
+
+set ppdir = `echo ${in_data_dir} | sed 's|\(.*pp\).*|\1|'`
+
+/home/oar.gfdl.mdtf/mdtf/MDTF-diagnostics/tools/run-MDTF/run-MDTF.sh -i ${ppdir} -o ${out_dir}/mdtf -s $yr1 -e $yr2
+
@@ -0,0 +1,31 @@
+# script to make the MDTF runtime configs for each realm/freq to run requested PODs
+import sys
+import json
+import copy
+
+# load pod information
+with open(sys.argv[2]) as f:
+    pods = json.load(f)
+
+# load template config information
+with open(sys.argv[1]+'template_config.jsonc') as f:
+    template_config = json.load(f)    
+
+# create dict object for each pod
+config_files = {}
+for p in pods:
+    config_name = f'{p}_config'
+    config_realm = pods[p]['realm']
+    config_files[config_name] = copy.deepcopy(template_config)
+    config_files[config_name]['case_list']['case_name']['realm'] = config_realm
+    config_files[config_name]['case_list'][config_realm] = config_files[config_name]['case_list'].pop('case_name')
+
+# add pods to cooresponding config file
+for p in pods:
+    config_file = f"{p}_config"    
+    config_files[config_file]['pod_list'].append(p) 
+
+#write out config files
+for c in config_files:
+    with open(sys.argv[1]+f"config/{c}.jsonc", "w") as f:
+        f.write(json.dumps(config_files[c], indent=2))     
@@ -0,0 +1,28 @@
+# script to create a simple index.html for automated mdtf runs
+import sys
+import os
+import json
+
+out_dir = sys.argv[1]
+
+# load pod information
+with open(sys.argv[2]) as f:
+    pods = json.load(f)
+
+index_file = open(f'{out_dir}/index.html', 'a')
+
+pod_htmls = {}
+pods = [p for p in pods]
+mdtf_outputs = [os.path.join(out_dir, d) for d in os.listdir(out_dir) if 'MDTF_output' in d]
+
+for d in mdtf_outputs:
+    list_d = os.listdir(d)
+    for p in pods:
+        if p in list_d:
+            pod_dir = os.path.join(d, p)
+            file_path = os.path.join(pod_dir, f'{p}.html')
+            if os.path.exists(file_path):
+                print(p)
+                index_file.write(f'<a href="{file_path}"> {p} </a>')
+
+index_file.close()
@@ -0,0 +1,11 @@
+<HTML>
+<HEAD>
+<meta http-equiv="expires" content="Fri, 01 Sep 00:00:00 EST 2025">
+<TITLE>MDTF-Diagnostics</TITLE>
+</HEAD>
+<BODY BGCOLOR="white">
+<IMG SRC="mdtf_diag_banner.png" border=0 usemap="#logo">
+<br clear=left>
+<p> Completed PODs </p>
+<HR SIZE=2 WIDTH="100%" ALIGN="LEFT" NOSHADE>
+