Working v3 test cfg

Author: forsyth2

tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg

@@ -9,10 +9,10 @@ infer_section_parameters = False
 input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_pcmdi_diags_20250925_try4"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id_pcmdi_diags_20250926"
 years = "1985:1989:2",
 
 [climo]
@@ -48,6 +48,7 @@ walltime = "00:30:00"
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
+  years = "1985:1995:2", # Need 10 years for pcmdi_diags task
 
   [[ atm_daily_180x360_aave ]]
   active = False
@@ -100,6 +101,7 @@ walltime = "00:30:00"
   [[ atm_monthly_180x360_aave ]]
   input_files = "eam.h0"
   ts_subsection = "atm_monthly_180x360_aave"
+  years = "1985:1995:2", # Need 10 years for pcmdi_diags task
 
   [[ land_monthly ]]
   input_files = "elm.h0"
@@ -167,16 +169,16 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/"
   tag = "model_vs_model"
   ts_num_years_ref = 2
   # Reference paths
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/clim"
   # mvm streamflow only
   gauges_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/GSIM/GSIM_catchment_characteristics_all_1km2.csv"
-  reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/rof/native/ts/monthly"
+  reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/rof/native/ts/monthly"
   # mvm diurnal_cycle only
-  reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily"
+  reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily"
   # mvm "enso_diags", "qbo", "area_mean_time_series"
-  reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly"
+  reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly"
   # mvm tropical_subseasonal only
-  reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily"
+  reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily"
 
   [[ lnd_monthly_mvm_lnd ]]
   # Test model-vs-model using the same files as the reference
@@ -192,7 +194,7 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/"
   tag = "model_vs_model"
   ts_num_years_ref = 2
   # Reference paths
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/lnd/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/lnd/180x360_aave/clim"
 
 [mpas_analysis]
 active = False
@@ -251,16 +253,16 @@ num_workers = 24
 obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series'
 pcmdi_debug = False
 ts_num_years = 2
-ts_years = "1985-1986",
+ts_years = "1985-1994",
 
   [[ mean_climate ]]
   active = True
   current_set = "mean_climate" # Can be inferred if needed
   # Default for all variables or specify source for each variable separately.
   obs_sets = "default"
-  ref_final_yr = 1986
+  ref_final_yr = 1994
   ref_start_yr = 1985
-  ref_years = "1985-1986",
+  ref_years = "1985-1994",
   # Variables in current pcmdi-cmip list
   # rltcre and rstcre are derived variables as they are not included in cmip_ts
   vars = "pr,rlut,rtmt,tas,tauu,tauv,ts"
@@ -276,9 +278,9 @@ ts_years = "1985-1986",
   current_set = "variability_modes_cpl" # Can be inferred if needed
   # See zppy/templates/pcmdi_diags/reference_alias.json for options
   obs_sets = "alternate1" #"default"
-  ref_final_yr = 1986
+  ref_final_yr = 1994
   ref_start_yr = 1985
-  ref_years = "1985-1986",
+  ref_years = "1985-1994",
   vars = "ts"
   # Name of modes varibility
   cpl_modes = "PDO,NPGO,AMO"
@@ -294,12 +296,12 @@ ts_years = "1985-1986",
   # See zppy/templates/pcmdi_diags/reference_alias.json for options
   # Default for all variables
   obs_sets = "alternate2" # "default"
-  ref_final_yr = 1986
+  ref_final_yr = 1994
   ref_start_yr = 1985
-  ref_years = "1985-1986",
+  ref_years = "1985-1994",
   vars = "psl"
   # Varibility modes
-  atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1" # PSA2 requires 3 years of data
+  atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2"
   frequency = 'mo'
   landmask = False
   ModUnitsAdjust = '(True,"divide",100.0)'
@@ -309,7 +311,8 @@ ts_years = "1985-1986",
   [[ synthetic_plots ]]
   active = True
   current_set = "synthetic_plots" # Can be inferred if needed
+  vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" # MUST match mean_climate's vars
   # clim_period, emov_period, enso_period:
-  figure_sets_period = "1985-1986,1985-1986,1985-1986"
+  figure_sets_period = "1985-1994,1985-1994,1985-1994"
   cmip_clim_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/mean_climate
   cmip_movs_dir = /lcrc/group/e3sm/diagnostics/pcmdi_data/metrics_data/variability_modes

tests/integration/generated/update_weekly_expected_files_chrysalis.sh

@@ -35,17 +35,17 @@ do
     # Copy output so you don't have to rerun zppy to generate the output.
     if [[ "${test_name,,}" =~ "v2" ]]; then
       # We need the v2 case name
-      cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250925_try4/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name}
+      cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250926/v2.LR.historical_0201 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name}
     else
       # We need the v3 case name
-      cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name}
+      cp -r /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_${test_name}_www/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051 /lcrc/group/e3sm/public_html/zppy_test_resources/expected_${test_name}
     fi
 
     # test_bundles.py also needs the bash files transferred.
     # Note that for legacy cfgs, we're only testing test_images.py
     if [[ "${test_name,,}" == "bundles" ]]; then
       mkdir -p /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
-      cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_pcmdi_diags_20250925_try4/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
+      cp -r /lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id_pcmdi_diags_20250926/v3.LR.historical_0051/post/scripts/bundle*.bash /lcrc/group/e3sm/public_html/zppy_test_resources/expected_bundles/bundle_files
     fi
 
     zppy_top_level=$(pwd)

tests/integration/template_weekly_comprehensive_v3.cfg

@@ -48,6 +48,7 @@ walltime = "00:30:00"
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
+  years = "1985:1995:2", # Need 10 years for pcmdi_diags task
 
   [[ atm_daily_180x360_aave ]]
   active = #expand active_e3sm_diags#
@@ -100,6 +101,7 @@ walltime = "00:30:00"
   [[ atm_monthly_180x360_aave ]]
   input_files = "eam.h0"
   ts_subsection = "atm_monthly_180x360_aave"
+  years = "1985:1995:2", # Need 10 years for pcmdi_diags task
 
   [[ land_monthly ]]
   input_files = "elm.h0"
@@ -251,16 +253,16 @@ num_workers = 24
 obs_ts = '/lcrc/soft/climate/e3sm_diags_data/obs_for_e3sm_diags/time-series'
 pcmdi_debug = False
 ts_num_years = 2
-ts_years = "1985-1986",
+ts_years = "1985-1994",
 
   [[ mean_climate ]]
   active = True
   current_set = "mean_climate" # Can be inferred if needed
   # Default for all variables or specify source for each variable separately.
   obs_sets = "default"
-  ref_final_yr = 1986
+  ref_final_yr = 1994
   ref_start_yr = 1985
-  ref_years = "1985-1986",
+  ref_years = "1985-1994",
   # Variables in current pcmdi-cmip list
   # rltcre and rstcre are derived variables as they are not included in cmip_ts
   vars = "pr,rlut,rtmt,tas,tauu,tauv,ts"
@@ -276,9 +278,9 @@ ts_years = "1985-1986",
   current_set = "variability_modes_cpl" # Can be inferred if needed
   # See zppy/templates/pcmdi_diags/reference_alias.json for options
   obs_sets = "alternate1" #"default"
-  ref_final_yr = 1986
+  ref_final_yr = 1994
   ref_start_yr = 1985
-  ref_years = "1985-1986",
+  ref_years = "1985-1994",
   vars = "ts"
   # Name of modes varibility
   cpl_modes = "PDO,NPGO,AMO"
@@ -294,12 +296,12 @@ ts_years = "1985-1986",
   # See zppy/templates/pcmdi_diags/reference_alias.json for options
   # Default for all variables
   obs_sets = "alternate2" # "default"
-  ref_final_yr = 1986
+  ref_final_yr = 1994
   ref_start_yr = 1985
-  ref_years = "1985-1986",
+  ref_years = "1985-1994",
   vars = "psl"
   # Varibility modes
-  atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1" # PSA2 requires 3 years of data
+  atm_modes = "NAM,NAO,PNA,NPO,SAM,PSA1,PSA2"
   frequency = 'mo'
   landmask = False
   ModUnitsAdjust = '(True,"divide",100.0)'
@@ -309,7 +311,8 @@ ts_years = "1985-1986",
   [[ synthetic_plots ]]
   active = True
   current_set = "synthetic_plots" # Can be inferred if needed
+  vars = "pr,rlut,rtmt,tas,tauu,tauv,ts" # MUST match mean_climate's vars
   # clim_period, emov_period, enso_period:
-  figure_sets_period = "1985-1986,1985-1986,1985-1986"
+  figure_sets_period = "1985-1994,1985-1994,1985-1994"
   cmip_clim_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/mean_climate
   cmip_movs_dir = #expand diagnostics_base_path#/pcmdi_data/metrics_data/variability_modes

tests/integration/utils.py

@@ -21,7 +21,7 @@
     "tasks_to_run": [
         "pcmdi_diags",
     ],
-    "unique_id": "unique_id_pcmdi_diags_20250925_try4",
+    "unique_id": "unique_id_pcmdi_diags_20250926",
 }
 
 # Multi-machine testing #########################################################

zppy/defaults/default.ini

@@ -432,7 +432,7 @@ ref_start_yr = string(default="")
 # For other plots, zppy will set it to match `years`, if not specified
 ref_years = string_list(default=list(""))
 # Variables to be used by the pcmdi diagnostics
-# mean_climate: vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500"
+# mean_climate AND synthetic_plots: vars = "pr,prw,psl,rlds,rldscs,rltcre,rstcre,rlut,rlutcs,rsds,rsdscs,rsdt,rsus,rsuscs,rlus,rsut,rtmt,sfcWind,tas,tauu,tauv,ts,ta-200,ta-850,ua-200,ua-850,va-200,va-850,zg-500"
 # variability_modes_atm: vars = "psl"
 # variability_modes_cpl: vars = "ts"
 # enso: vars = "psl,pr,prsn,ts,tas,tauu,tauv,hflx,hfss,rlds,rsds,rlus,rlut,rsdt"