Attempt to multithread the image tests

Author: forsyth2

tests/integration/generated/test_weekly_bundles_chrysalis.cfg

@@ -27,16 +27,16 @@
 case = "v3.LR.historical_0051"
 constraint = ""
 dry_run = "False"
-environment_commands = ""
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id/v3.LR.historical_0051"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/update_test_images_20251021_try2/v3.LR.historical_0051"
 partition = "compute"
 qos = "regular"
 walltime = "07:00:00"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_bundles_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_bundles_www/update_test_images_20251021_try2"
 
 [bundle]
 
@@ -118,7 +118,7 @@ years = "1985:1989:2",
 
 [e3sm_diags]
 active = True
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 grid = '180x360_aave'
 ref_final_yr = 1989
 ref_start_yr = 1985
@@ -141,7 +141,7 @@ years = "1985:1989:2",
   ref_name = "v3.LR.historical_0051"
   ref_start_yr = 1985
   ref_years = "1985-1986",
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_bundles_output/update_test_images_20251021_try2/v3.LR.historical_0051/post/atm/180x360_aave/clim"
   run_type = "model_vs_model"
   # TODO: Add "tc_analysis" back in after empty dat is resolved.
   sets = "polar","enso_diags","streamflow",
@@ -155,7 +155,7 @@ years = "1985:1989:2",
 [global_time_series]
 active = True
 bundle = "bundle2"
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 plots_original="net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"

tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg

@@ -2,15 +2,15 @@
 case = "v2.LR.historical_0201"
 constraint = ""
 dry_run = "False"
-environment_commands = ""
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 fail_on_dependency_skip = True
 input = /lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/update_test_images_20251021_try2/v2.LR.historical_0201"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v2_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v2_www/update_test_images_20251021_try2"
 years = "1980:1984:2",
 
 [climo]
@@ -107,7 +107,7 @@ walltime = "00:30:00"
 active = True
 climo_diurnal_frequency = "diurnal_8xdaily"
 climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 grid = '180x360_aave'
 multiprocessing = True
 num_workers = 8
@@ -140,7 +140,7 @@ years = "1982:1984:2",
   partition = "compute"
   qos = "regular"
   ref_name = "v2.LR.historical_0201"
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/update_test_images_20251021_try2/v2.LR.historical_0201/post/atm/180x360_aave/clim"
   run_type = "model_vs_model"
   short_ref_name = "same simulation"
   swap_test_ref = False
@@ -157,7 +157,7 @@ years = "1982:1984:2",
   partition = "compute"
   qos = "regular"
   ref_name = "v2.LR.historical_0201"
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/update_test_images_20251021_try2/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
   run_type = "model_vs_model"
   sets = "lat_lon_land",
   short_ref_name = "same simulation"
@@ -181,7 +181,7 @@ walltime = "00:30:00"
 [global_time_series]
 active = True
 climo_years ="1980-1984", "1985-1990",
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 experiment_name = "v2.LR.historical_0201"
 figstr = "v2.LR.historical_0201"
 moc_file=mocTimeSeries_1980-1990.nc

tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg

@@ -2,17 +2,17 @@
 case = "v3.LR.historical_0051"
 constraint = ""
 dry_run = "False"
-environment_commands = ""
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 fail_on_dependency_skip = True
 infer_path_parameters = False
 infer_section_parameters = False
 input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/update_test_images_20251021_try2/v3.LR.historical_0051"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v3_www/update_test_images_20251021_try2"
 years = "1985:1989:2",
 
 [climo]
@@ -120,7 +120,7 @@ active = True
 climo_diurnal_frequency = "diurnal_8xdaily"
 climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
 climo_subsection = "atm_monthly_180x360_aave"
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 grid = '180x360_aave'
 multiprocessing = True
 num_workers = 8
@@ -172,16 +172,16 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/"
   tag = "model_vs_model"
   ts_num_years_ref = 2
   # Reference paths
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/update_test_images_20251021_try2/v3.LR.historical_0051/post/atm/180x360_aave/clim"
   # mvm streamflow only
   gauges_path = "/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/GSIM/GSIM_catchment_characteristics_all_1km2.csv"
-  reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/rof/native/ts/monthly"
+  reference_data_path_ts_rof = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/update_test_images_20251021_try2/v3.LR.historical_0051/post/rof/native/ts/monthly"
   # mvm diurnal_cycle only
-  reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily"
+  reference_data_path_climo_diurnal = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/update_test_images_20251021_try2/v3.LR.historical_0051/post/atm/180x360_aave/clim_diurnal_8xdaily"
   # mvm "enso_diags", "qbo", "area_mean_time_series"
-  reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly"
+  reference_data_path_ts = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/update_test_images_20251021_try2/v3.LR.historical_0051/post/atm/180x360_aave/ts/monthly"
   # mvm tropical_subseasonal only
-  reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily"
+  reference_data_path_ts_daily = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/update_test_images_20251021_try2/v3.LR.historical_0051/post/atm/180x360_aave/ts/daily"
 
   [[ lnd_monthly_mvm_lnd ]]
   # Test model-vs-model using the same files as the reference
@@ -197,7 +197,7 @@ tc_obs = "/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/"
   tag = "model_vs_model"
   ts_num_years_ref = 2
   # Reference paths
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/unique_id/v3.LR.historical_0051/post/lnd/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v3_output/update_test_images_20251021_try2/v3.LR.historical_0051/post/lnd/180x360_aave/clim"
 
 [mpas_analysis]
 active = True
@@ -215,7 +215,7 @@ walltime = "00:30:00"
 [global_time_series]
 active = True
 climo_years = "1985-1989", "1990-1995",
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 #moc_file=mocTimeSeries_1985-1995.nc
@@ -314,7 +314,7 @@ years = "1985:1989:4"
 
 [pcmdi_diags]
 active = True
-environment_commands_secondary = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands_secondary = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 walltime = "2:00:00"
 model_name = 'e3sm.historical.v3-LR.0051'
 model_tableID = 'Amon'

tests/integration/generated/test_weekly_legacy_3.0.0_bundles_chrysalis.cfg

@@ -27,16 +27,16 @@
 case = "v3.LR.historical_0051"
 constraint = ""
 dry_run = "False"
-environment_commands = ""
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 input = /lcrc/group/e3sm2/ac.wlin//E3SMv3/v3.LR.historical_0051
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
 # To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_bundles.py
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_bundles_output/unique_id/v3.LR.historical_0051"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_bundles_output/update_test_images_20251021_try2/v3.LR.historical_0051"
 partition = "compute"
 qos = "regular"
 walltime = "07:00:00"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_legacy_3.0.0_bundles_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_legacy_3.0.0_bundles_www/update_test_images_20251021_try2"
 
 [bundle]
 
@@ -118,7 +118,7 @@ years = "1985:1989:2",
 
 [e3sm_diags]
 active = True
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 grid = '180x360_aave'
 ref_final_yr = 1989
 ref_start_yr = 1985
@@ -141,7 +141,7 @@ years = "1985:1989:2",
   ref_name = "v3.LR.historical_0051"
   ref_start_yr = 1985
   ref_years = "1985-1986",
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_bundles_output/unique_id/v3.LR.historical_0051/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_bundles_output/update_test_images_20251021_try2/v3.LR.historical_0051/post/atm/180x360_aave/clim"
   run_type = "model_vs_model"
   # TODO: Add "tc_analysis" back in after empty dat is resolved.
   sets = "polar","enso_diags","streamflow",
@@ -155,7 +155,7 @@ years = "1985:1989:2",
 [global_time_series]
 active = True
 bundle = "bundle2"
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 plots_original="net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"

tests/integration/generated/test_weekly_legacy_3.0.0_comprehensive_v2_chrysalis.cfg

@@ -2,15 +2,15 @@
 case = "v2.LR.historical_0201"
 constraint = ""
 dry_run = "False"
-environment_commands = ""
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 fail_on_dependency_skip = True
 input = /lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201
 input_subdir = archive/atm/hist
 mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
-output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/unique_id/v2.LR.historical_0201"
+output = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/update_test_images_20251021_try2/v2.LR.historical_0201"
 partition = "debug"
 qos = "regular"
-www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v2_www/unique_id"
+www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v2_www/update_test_images_20251021_try2"
 years = "1980:1984:2",
 
 [climo]
@@ -107,7 +107,7 @@ walltime = "00:30:00"
 active = True
 climo_diurnal_frequency = "diurnal_8xdaily"
 climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 grid = '180x360_aave'
 multiprocessing = True
 num_workers = 8
@@ -140,7 +140,7 @@ years = "1982:1984:2",
   partition = "compute"
   qos = "regular"
   ref_name = "v2.LR.historical_0201"
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/unique_id/v2.LR.historical_0201/post/atm/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/update_test_images_20251021_try2/v2.LR.historical_0201/post/atm/180x360_aave/clim"
   run_type = "model_vs_model"
   short_ref_name = "same simulation"
   swap_test_ref = False
@@ -157,7 +157,7 @@ years = "1982:1984:2",
   partition = "compute"
   qos = "regular"
   ref_name = "v2.LR.historical_0201"
-  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/unique_id/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
+  reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_legacy_3.0.0_comprehensive_v2_output/update_test_images_20251021_try2/v2.LR.historical_0201/post/lnd/180x360_aave/clim"
   run_type = "model_vs_model"
   sets = "lat_lon_land",
   short_ref_name = "same simulation"
@@ -181,7 +181,7 @@ walltime = "00:30:00"
 [global_time_series]
 active = True
 climo_years ="1980-1984", "1985-1990",
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.12.0rc2_chrysalis.sh"
 experiment_name = "v2.LR.historical_0201"
 figstr = "v2.LR.historical_0201"
 moc_file=mocTimeSeries_1980-1990.nc