Split global_time_series

forsyth2 · forsyth2 · commit 81bbae5a6dba · 2025-06-05T19:27:16.000-05:00
diff --git a/conda/dev.yml b/conda/dev.yml
@@ -1,3 +1,9 @@
+# Conda development environment for testing local source code changes to zppy before merging them to production (main branch).
+# To set up:
+# conda clean --all --y
+# conda env create -f conda/dev.yml -n env_name
+# pre-commit run --all-files
+# pip install .
 name: zppy_dev
 channels:
   - conda-forge
diff --git a/tests/integration/generated/test_min_case_deprecated_parameters_chrysalis.cfg b/tests/integration/generated/test_min_case_deprecated_parameters_chrysalis.cfg
@@ -36,7 +36,7 @@ walltime = "00:30:00"
 active = True
 atmosphere_only = "False" # DEPRECATED in https://github.com/E3SM-Project/zppy/pull/654
 climo_years = "1985-1989", "1990-1995",
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 moc_file=mocTimeSeries_1985-1995.nc
diff --git a/tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg
@@ -33,7 +33,7 @@ years = "1985:1995:5",
 
 [global_time_series]
 active = True
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 plots_original=""
diff --git a/tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg
@@ -39,7 +39,7 @@ walltime = "00:30:00"
 [global_time_series]
 active = True
 climo_years = "1985-1989", "1990-1995",
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 moc_file=mocTimeSeries_1985-1995.nc
diff --git a/tests/integration/generated/test_min_case_global_time_series_original_8_missing_ocn_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_original_8_missing_ocn_chrysalis.cfg
@@ -26,7 +26,7 @@ walltime = "00:30:00"
 [global_time_series]
 active = True
 climo_years = "1985-1989", "1990-1995",
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 # plots_original -- use the default vars,
diff --git a/tests/integration/generated/test_min_case_global_time_series_original_8_no_ocn_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_original_8_no_ocn_chrysalis.cfg
@@ -25,7 +25,7 @@ walltime = "00:30:00"
 
 [global_time_series]
 active = True
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 plots_original="net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"
diff --git a/tests/integration/generated/test_min_case_global_time_series_viewers_all_land_variables_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_viewers_all_land_variables_chrysalis.cfg
@@ -33,7 +33,7 @@ years = "1985:1995:5",
 
 [global_time_series]
 active = True
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 make_viewer = True
diff --git a/tests/integration/generated/test_min_case_global_time_series_viewers_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_viewers_chrysalis.cfg
@@ -33,7 +33,7 @@ years = "1985:1995:5",
 
 [global_time_series]
 active = True
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 make_viewer = True
diff --git a/tests/integration/generated/test_min_case_global_time_series_viewers_original_8_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_viewers_original_8_chrysalis.cfg
@@ -39,7 +39,7 @@ walltime = "00:30:00"
 [global_time_series]
 active = True
 climo_years = "1985-1989", "1990-1995",
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 make_viewer = True
diff --git a/tests/integration/generated/test_min_case_global_time_series_viewers_original_atm_plus_land_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_viewers_original_atm_plus_land_chrysalis.cfg
@@ -32,7 +32,7 @@ years = "1985:1995:5",
 
 [global_time_series]
 active = True
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 make_viewer = True
diff --git a/tests/integration/generated/test_min_case_global_time_series_viewers_undefined_variables_chrysalis.cfg b/tests/integration/generated/test_min_case_global_time_series_viewers_undefined_variables_chrysalis.cfg
@@ -33,7 +33,7 @@ years = "1985:1995:5",
 
 [global_time_series]
 active = True
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 make_viewer = True
diff --git a/tests/integration/generated/test_weekly_bundles_chrysalis.cfg b/tests/integration/generated/test_weekly_bundles_chrysalis.cfg
@@ -151,7 +151,7 @@ years = "1985:1989:2",
 [global_time_series]
 active = True
 bundle = "bundle2"
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 plots_original="net_toa_flux_restom,global_surface_air_temperature,toa_radiation,net_atm_energy_imbalance,net_atm_water_imbalance"
diff --git a/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg
@@ -175,7 +175,7 @@ walltime = "00:30:00"
 [global_time_series]
 active = True
 climo_years ="1980-1984", "1985-1990",
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v2.LR.historical_0201"
 figstr = "v2.LR.historical_0201"
 moc_file=mocTimeSeries_1980-1990.nc
diff --git a/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg b/tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg
@@ -16,7 +16,7 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_co
 years = "1985:1989:2",
 
 [climo]
-active = True
+active = False
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
@@ -43,17 +43,20 @@ active = True
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
+  active = False
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
 
   [[ atm_daily_180x360_aave ]]
+  active = False
   frequency = "daily"
   input_files = "eam.h1"
   input_subdir = "archive/atm/hist"
   vars = "PRECT"
 
   [[ rof_monthly ]]
+  active = False
   extra_vars = 'areatotal2'
   frequency = "monthly"
   input_files = "mosart.h0"
@@ -63,13 +66,15 @@ walltime = "00:30:00"
 
   [[ atm_monthly_glb ]]
   # Note global average won't work for 3D variables.
+  active = True
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   mapping_file = "glb"
   years = "1985:1995:5",
 
   [[ lnd_monthly_glb ]]
+  active = True
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
@@ -78,6 +83,7 @@ walltime = "00:30:00"
   years = "1985:1995:5",
 
   [[ land_monthly ]]
+  active = False
   extra_vars = "landfrac"
   frequency = "monthly"
   input_files = "elm.h0"
@@ -86,7 +92,7 @@ walltime = "00:30:00"
   vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 [e3sm_to_cmip]
-active = True
+active = False
 frequency = "monthly"
 ts_num_years=2
 walltime = "00:30:00"
@@ -105,7 +111,7 @@ walltime = "00:30:00"
 # walltime = "00:30:00"
 
 [e3sm_diags]
-active = True
+active = False
 climo_diurnal_frequency = "diurnal_8xdaily"
 climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
 climo_subsection = "atm_monthly_180x360_aave"
@@ -205,7 +211,7 @@ walltime = "00:30:00"
 [global_time_series]
 active = True
 climo_years = "1985-1989", "1990-1995",
-environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
+environment_commands = "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603"
 experiment_name = "v3.LR.historical_0051"
 figstr = "v3.LR.historical_0051"
 moc_file=mocTimeSeries_1985-1995.nc
@@ -216,7 +222,7 @@ walltime = "00:30:00"
 years = "1985-1995",
 
 [ilamb]
-active = True
+active = False
 e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave"
 e3sm_to_cmip_land_subsection = "land_monthly"
 ilamb_obs = "/lcrc/group/e3sm/diagnostics/ilamb_data"
diff --git a/tests/integration/image_checker.py b/tests/integration/image_checker.py
@@ -61,6 +61,9 @@ def set_up_and_run_image_checker(
         "diff_dir": f"{actual_images_dir}image_check_failures_{cfg_specifier}{diff_dir_suffix}",
         "expected_images_list": f"{expansions['expected_dir']}image_list_expected_{cfg_specifier}.txt",
     }
+    print("Image checking dict:")
+    for key in d:
+        print(f"{key}: {d[key]}")
     parameters: Parameters = Parameters(d)
     for task in task_list:
         test_results = check_images(parameters, task)
diff --git a/tests/integration/template_weekly_comprehensive_v3.cfg b/tests/integration/template_weekly_comprehensive_v3.cfg
@@ -16,7 +16,7 @@ www = "#expand user_www#zppy_weekly_comprehensive_v3_www/#expand unique_id#"
 years = "1985:1989:2",
 
 [climo]
-active = True
+active = False
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
@@ -43,17 +43,20 @@ active = True
 walltime = "00:30:00"
 
   [[ atm_monthly_180x360_aave ]]
+  active = False
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
 
   [[ atm_daily_180x360_aave ]]
+  active = False
   frequency = "daily"
   input_files = "eam.h1"
   input_subdir = "archive/atm/hist"
   vars = "PRECT"
 
   [[ rof_monthly ]]
+  active = False
   extra_vars = 'areatotal2'
   frequency = "monthly"
   input_files = "mosart.h0"
@@ -63,13 +66,15 @@ walltime = "00:30:00"
 
   [[ atm_monthly_glb ]]
   # Note global average won't work for 3D variables.
+  active = True
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
   mapping_file = "glb"
   years = "1985:1995:5",
 
   [[ lnd_monthly_glb ]]
+  active = True
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
@@ -78,6 +83,7 @@ walltime = "00:30:00"
   years = "1985:1995:5",
 
   [[ land_monthly ]]
+  active = False
   extra_vars = "landfrac"
   frequency = "monthly"
   input_files = "elm.h0"
@@ -86,7 +92,7 @@ walltime = "00:30:00"
   vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
 
 [e3sm_to_cmip]
-active = True
+active = False
 frequency = "monthly"
 ts_num_years=2
 walltime = "00:30:00"
@@ -105,7 +111,7 @@ walltime = "00:30:00"
 # walltime = "00:30:00"
 
 [e3sm_diags]
-active = True
+active = False
 climo_diurnal_frequency = "diurnal_8xdaily"
 climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
 climo_subsection = "atm_monthly_180x360_aave"
@@ -216,7 +222,7 @@ walltime = "00:30:00"
 years = "1985-1995",
 
 [ilamb]
-active = True
+active = False
 e3sm_to_cmip_atm_subsection = "atm_monthly_180x360_aave"
 e3sm_to_cmip_land_subsection = "land_monthly"
 ilamb_obs = "#expand diagnostics_base_path#/ilamb_data"
diff --git a/tests/integration/test_images.py b/tests/integration/test_images.py
@@ -45,15 +45,10 @@ def test_images():
             V3_CASE_NAME,
             expansions,
             diff_dir_suffix,
-            ["e3sm_diags", "mpas_analysis", "global_time_series", "ilamb"],
-            test_results_dict,
-        )
-        set_up_and_run_image_checker(
-            "bundles",
-            V3_CASE_NAME,
-            expansions,
-            diff_dir_suffix,
-            ["e3sm_diags", "global_time_series", "ilamb"],  # No mpas_analysis
+            [
+                "mpas_analysis",
+                "global_time_series",
+            ],  # ["e3sm_diags", "mpas_analysis", "global_time_series", "ilamb"],
             test_results_dict,
         )
 
diff --git a/tests/integration/utils.py b/tests/integration/utils.py
@@ -3,6 +3,12 @@
 
 from mache import MachineInfo
 
+# To run:
+# `pip install .` latest code into conda env
+# Update UNIQUE_ID and any necessary environments below
+# python tests/integration/utils.py
+# zppy -c <generated cfg>
+
 UNIQUE_ID = "unique_id"
 
 # Example testing workflow ####################################################
@@ -96,7 +102,7 @@ def get_chyrsalis_expansions(config):
         "diags_walltime": "5:00:00",
         "environment_commands_test": "",
         "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/",
-        "global_time_series_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
+        "global_time_series_environment_commands": "source /gpfs/fs1/home/ac.forsyth2/miniforge3/etc/profile.d/conda.sh; conda activate zi-gts-split-20250603",
         "mpas_analysis_walltime": "00:30:00",
         "partition_long": "compute",
         "partition_short": "debug",
diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini
@@ -348,6 +348,24 @@ regions = string(default="glb,n,s")
 ts_years = string_list(default=list(""))
 # `years = "1-100",` would plot years 1 to 100 on the graphs.
 
+  [[__many__]]
+  climo_years = string_list(default=None)
+  color = string(default=None)
+  experiment_name = string(default=None)
+  figstr = string(default=None)
+  input_subdir = string(default=None)
+  make_viewer = boolean(default=None)
+  moc_file = string(default=None)
+  ncols = integer(default=None)
+  nrows = integer(default=None)
+  plots_original = string(default=None)
+  plots_atm = string(default=None)
+  plots_ice = string(default=None)
+  plots_lnd = string(default=None)
+  plots_ocn = string(default=None)
+  regions = string(default=None)
+  ts_years = string_list(default=None)
+
 [ilamb]
 cfg = string(default="inclusions/ilamb/cmip.cfg")
 # Name of the `[e3sm_to_cmip]` atm subtask to depend on
diff --git a/zppy/templates/global_time_series.bash b/zppy/templates/global_time_series.bash
@@ -5,9 +5,14 @@
 
 # Generate global time series plots
 ################################################################################
+{% if subsection != None %}
+results_dir={{ subsection }}/{{ prefix }}_results
+zi-global-time-series --subsection {{ subsection }} --use_ocn {{ use_ocn }} --input {{ input }} --input_subdir {{ input_subdir }} --moc_file {{ moc_file }} --case_dir {{ output }} --experiment_name {{ experiment_name }} --figstr {{ figstr }} --color {{ color }} --ts_num_years {{ ts_num_years }} --plots_original {{ plots_original }} --plots_atm {{ plots_atm }} --plots_ice {{ plots_ice }} --plots_lnd {{ plots_lnd }} --plots_ocn {{ plots_ocn }} --nrows {{ nrows }} --ncols {{ ncols }} --results_dir ${results_dir} --regions {{ regions }} --make_viewer {{ make_viewer }} --start_yr {{ year1 }} --end_yr {{ year2 }}
+{%- else %}
 results_dir={{ prefix }}_results
+zi-global-time-series --subsection "no_subsection" --use_ocn {{ use_ocn }} --input {{ input }} --input_subdir {{ input_subdir }} --moc_file {{ moc_file }} --case_dir {{ output }} --experiment_name {{ experiment_name }} --figstr {{ figstr }} --color {{ color }} --ts_num_years {{ ts_num_years }} --plots_original {{ plots_original }} --plots_atm {{ plots_atm }} --plots_ice {{ plots_ice }} --plots_lnd {{ plots_lnd }} --plots_ocn {{ plots_ocn }} --nrows {{ nrows }} --ncols {{ ncols }} --results_dir ${results_dir} --regions {{ regions }} --make_viewer {{ make_viewer }} --start_yr {{ year1 }} --end_yr {{ year2 }}
+{%- endif %}
 
-zi-global-time-series --use_ocn {{ use_ocn }} --input {{ input }} --input_subdir {{ input_subdir }} --moc_file {{ moc_file }} --case_dir {{ output }} --experiment_name {{ experiment_name }} --figstr {{ figstr }} --color {{ color }} --ts_num_years {{ ts_num_years }} --plots_original {{ plots_original }} --plots_atm {{ plots_atm }} --plots_ice {{ plots_ice }} --plots_lnd {{ plots_lnd }} --plots_ocn {{ plots_ocn }} --nrows {{ nrows }} --ncols {{ ncols }} --results_dir ${results_dir} --regions {{ regions }} --make_viewer {{ make_viewer }} --start_yr {{ year1 }} --end_yr {{ year2 }}
 
 results_dir_absolute_path={{ scriptDir }}/${results_dir}
 # We are already in scriptDir so we don't have to copy files over to results_dir_absolute_path
