Skip to content

Commit e2fcc3a

Browse files
tomvothecodertomvothecoder
authored
Fix --no_viewer option breaking (#943)
1 parent ee134b8 commit e2fcc3a

File tree

4 files changed

+236
-7
lines changed

4 files changed

+236
-7
lines changed

‎auxiliary_tools/cdat_regression_testing/942-no-viewer/qa.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import os
2+
from e3sm_diags.parameter.core_parameter import CoreParameter
3+
from e3sm_diags.parameter.tc_analysis_parameter import TCAnalysisParameter
4+
from e3sm_diags.run import Run, runner
5+
6+
simulations = [
7+
"tc-v1.HR.0026_0035",
8+
"tc-v2.LR.2000_2014",
9+
"tc-v3.HR.0006_0025",
10+
"tc-v3.LR.2000_2014",
11+
]
12+
sim_names = [
13+
"theta.20180906.branch_noCNT.A_WCYCL1950S_CMIP6_HR.ne120_oRRS18v3_ICG",
14+
"v2.LR.historical_0101",
15+
"20240609.piCtl.ne120pg2_r025_RRSwISC6to18E3r5.chrysalis.test1",
16+
"extendedOutput.v3.LR.historical_0101",
17+
]
18+
19+
data_path = "/global/homes/c/chengzhu/tests/tc_analysis/"
20+
21+
for idx, sim in enumerate(simulations):
22+
print(sim)
23+
# runner = Run()
24+
25+
param = CoreParameter()
26+
param.multiprocessing = True
27+
param.test_data_path = data_path + sim
28+
param.test_name = sim_names[idx]
29+
param.test_start_yr = sim.split(".")[-1][0:4]
30+
param.test_end_yr = sim.split(".")[-1][5:9]
31+
32+
param.reference_data_path = (
33+
"/global/cfs/cdirs/e3sm/diagnostics/observations/Atm/tc-analysis"
34+
)
35+
param.ref_start_yr = "1979"
36+
param.ref_end_yr = "2018"
37+
38+
prefix = "/global/cfs/cdirs/e3sm/www/vo13/tc_analysis_test/"
39+
param.results_dir = os.path.join(prefix, sim)
40+
runner.sets_to_run = ["tc_analysis"]
41+
runner.run_diags([param])

‎auxiliary_tools/cdat_regression_testing/942-no-viewer/v2_run.cfg‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[#]
2+
sets = ["lat_lon"]
3+
case_id = "ERA5"
4+
variables = ["TREFHT"]
5+
regions = ["land"]
6+
ref_name = "ERA5"
7+
reference_name = "ERA5 Reanalysis"
8+
seasons=["ANN"]
9+
contour_levels = [-35, -30, -25, -20, -15, -10, -5, 0, 5, 10, 15, 20, 25, 30, 35, 40]
10+
diff_levels = [-12, -8, -4, -2, -1, -0.5, 0.5, 1, 2, 4, 8, 12]
11+
regrid_method = "bilinear"

‎auxiliary_tools/cdat_regression_testing/942-no-viewer/v2_run_script.py‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
"""
2+
Source: /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v2_www/test_pr651_both_commits_20250117/v2.LR.historical_0201/e3sm_diags/atm_monthly_180x360_aave/model_vs_obs_1982-1983/prov/e3sm.py
3+
4+
Webpage: https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v2_www/test_pr651_both_commits_20250117/v2.LR.historical_0201/e3sm_diags/atm_monthly_180x360_aave/model_vs_obs_1982-1983/prov/e3sm.py
5+
6+
Diffs: https://github.com/E3SM-Project/zppy/pull/651#issuecomment-2628445196
7+
/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_weekly_comprehensive_v2_www/test_pr651_both_commits_20250117/v2.LR.historical_0201/image_check_failures_comprehensive_v2/e3sm_diags/atm_monthly_180x360_aave/model_vs_obs_1982-1983
8+
9+
ERA5-TREFHT-ANN-land.png
10+
ERA5_ext-QREFHT-ANN-global.png
11+
ERA5_ext-U10-ANN-global.png
12+
GPCP_v3.2-PRECT-ANN-global.png
13+
HadISST_CL-SST-ANN-global.png
14+
HadISST_PD-SST-ANN-global.png
15+
HadISST_PI-SST-ANN-global.png
16+
MACv2-AODVIS-ANN-global.png
17+
MERRA2-OMEGA-850-ANN-global.png
18+
MERRA2-PSL-ANN-global.png
19+
MERRA2-T-850-ANN-global.png
20+
MERRA2-TAUXY-ANN-ocean.png
21+
MERRA2-TREFHT-ANN-land.png
22+
MERRA2-TREFMNAV-ANN-global.png
23+
MERRA2-TREFMXAV-ANN-global.png
24+
_MISRCOSP-CLDLOW_TAU1.3_9.4_MISR-ANN-global.png
25+
MISRCOSP-CLDLOW_TAU1.3_MISR-ANN-global.png
26+
MISRCOSP-CLDLOW_TAU9.4_MISR-ANN-global.png_
27+
OMI-MLS-TCO-ANN-60S60N.png
28+
ceres_ebaf_surface_v4.1-ALBEDO_SRF-ANN-global.png
29+
"""
30+
31+
import os
32+
import sys
33+
34+
import numpy
35+
from e3sm_diags.parameter.core_parameter import CoreParameter
36+
from e3sm_diags.parameter.diurnal_cycle_parameter import DiurnalCycleParameter
37+
from e3sm_diags.parameter.enso_diags_parameter import EnsoDiagsParameter
38+
from e3sm_diags.parameter.qbo_parameter import QboParameter
39+
from e3sm_diags.parameter.streamflow_parameter import StreamflowParameter
40+
from e3sm_diags.parameter.tc_analysis_parameter import TCAnalysisParameter
41+
from e3sm_diags.parameter.tropical_subseasonal_parameter import TropicalSubseasonalParameter
42+
43+
44+
from e3sm_diags.run import runner
45+
46+
short_name = 'v2.LR.historical_0201'
47+
test_ts = '/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/test_pr651_both_commits_20250117/v2.LR.historical_0201/post/atm/180x360_aave/ts/monthly/2yr'
48+
start_yr = int('1982')
49+
end_yr = int('1983')
50+
num_years = end_yr - start_yr + 1
51+
ref_start_yr = 1980
52+
53+
param = CoreParameter()
54+
55+
# Model
56+
# param.test_data_path = 'climo'
57+
param.test_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/test_pr651_both_commits_20250117/v2.LR.historical_0201/post/atm/180x360_aave/clim/2yr"
58+
param.test_name = 'v2.LR.historical_0201'
59+
param.short_test_name = short_name
60+
61+
# Ref
62+
63+
# Obs
64+
param.reference_data_path = '/lcrc/group/e3sm/diagnostics/observations/Atm/climatology/'
65+
param.save_netcdf = True
66+
67+
# Output dir
68+
# param.results_dir = 'model_vs_obs_1982-1983'
69+
param.results_dir = "/lcrc/group/e3sm/public_html/cdat-migration-fy24/942-no-viewer"
70+
71+
# Additional settings
72+
param.run_type = 'model_vs_obs'
73+
param.diff_title = 'Model - Observations'
74+
param.output_format = ['png']
75+
param.output_format_subplot = []
76+
param.multiprocessing = True
77+
param.num_workers = 8
78+
#param.fail_on_incomplete = True
79+
params = [param]
80+
81+
# Model land
82+
enso_param = EnsoDiagsParameter()
83+
enso_param.test_data_path = test_ts
84+
enso_param.test_name = short_name
85+
enso_param.test_start_yr = start_yr
86+
enso_param.test_end_yr = end_yr
87+
88+
# Obs
89+
enso_param.reference_data_path = '/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/'
90+
enso_param.ref_start_yr = ref_start_yr
91+
enso_param.ref_end_yr = ref_start_yr + 10
92+
93+
params.append(enso_param)
94+
trop_param = TropicalSubseasonalParameter()
95+
trop_param.test_data_path = '/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/test_pr651_both_commits_20250117/v2.LR.historical_0201/post/atm/180x360_aave/ts/daily/2yr'
96+
trop_param.test_name = short_name
97+
trop_param.test_start_yr = f'{start_yr:04}'
98+
trop_param.test_end_yr = f'{end_yr:04}'
99+
100+
# Obs
101+
trop_param.reference_data_path = '/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/'
102+
trop_param.ref_start_yr = 2001
103+
trop_param.ref_end_yr = 2010
104+
105+
params.append(trop_param)
106+
qbo_param = QboParameter()
107+
qbo_param.test_data_path = test_ts
108+
qbo_param.test_name = short_name
109+
qbo_param.test_start_yr = start_yr
110+
qbo_param.test_end_yr = end_yr
111+
qbo_param.ref_start_yr = ref_start_yr
112+
ref_end_yr = ref_start_yr + num_years - 1
113+
if (ref_end_yr <= 1981):
114+
qbo_param.ref_end_yr = ref_end_yr
115+
else:
116+
qbo_param.ref_end_yr = 1981
117+
118+
# Obs
119+
qbo_param.reference_data_path = '/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/'
120+
121+
params.append(qbo_param)
122+
dc_param = DiurnalCycleParameter()
123+
dc_param.test_data_path = 'climo_diurnal_8xdaily'
124+
dc_param.short_test_name = short_name
125+
# Plotting diurnal cycle amplitude on different scales. Default is True
126+
dc_param.normalize_test_amp = False
127+
128+
# Obs
129+
dc_param.reference_data_path = '/lcrc/group/e3sm/public_html/e3sm_diags_test_data/unit_test_complete_run/obs/climatology'
130+
131+
params.append(dc_param)
132+
streamflow_param = StreamflowParameter()
133+
streamflow_param.test_data_path = '/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/test_pr651_both_commits_20250117/v2.LR.historical_0201/post/rof/native/ts/monthly/2yr'
134+
streamflow_param.test_name = short_name
135+
streamflow_param.test_start_yr = start_yr
136+
streamflow_param.test_end_yr = end_yr
137+
138+
# Obs
139+
streamflow_param.reference_data_path = '/lcrc/group/e3sm/diagnostics/observations/Atm/time-series/'
140+
streamflow_param.ref_start_yr = "1986" # Streamflow gauge station data range from year 1986 to 1995
141+
streamflow_param.ref_end_yr = "1995"
142+
143+
params.append(streamflow_param)
144+
tc_param = TCAnalysisParameter()
145+
tc_param.test_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_weekly_comprehensive_v2_output/test_pr651_both_commits_20250117/v2.LR.historical_0201/post/atm/tc-analysis_1982_1983"
146+
tc_param.short_test_name = short_name
147+
tc_param.test_start_yr = "1982"
148+
tc_param.test_end_yr = "1983"
149+
150+
# Obs
151+
tc_param.reference_data_path = '/lcrc/group/e3sm/diagnostics/observations/Atm/tc-analysis/'
152+
# For model vs obs, the ref start and end year can be any four digit strings
153+
# For now, use all available years from obs by default
154+
tc_param.ref_start_yr = "1979"
155+
tc_param.ref_end_yr = "2018"
156+
157+
params.append(tc_param)
158+
159+
# Run
160+
cfg_path = "auxiliary_tools/cdat_regression_testing/942-no-viewer/v2_run.cfg"
161+
sys.argv.extend(["--diags", cfg_path, "--no_viewer"])
162+
163+
# runner.sets_to_run = ['lat_lon', 'zonal_mean_xy', 'zonal_mean_2d', 'polar', 'cosp_histogram', 'meridional_mean_2d', 'annual_cycle_zonal_mean', 'zonal_mean_2d_stratosphere', 'enso_diags', 'qbo', 'diurnal_cycle', 'streamflow', 'tc_analysis', 'tropical_subseasonal', 'aerosol_aeronet', 'aerosol_budget']
164+
165+
runner.sets_to_run = ['lat_lon']
166+
runner.run_diags(params)
167+

‎e3sm_diags/e3sm_diags_driver.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
#!/usr/bin/env python
2-
# The above line is needed for `test_all_sets.test_all_sets_mpl`.
3-
# Otherwise, OSError: [Errno 8] Exec format error: 'e3sm_diags_driver.py'.
41
from __future__ import annotations
52

63
import os
@@ -71,7 +68,7 @@ def get_default_diags_path(set_name, run_type, print_path=True):
7168
return pth
7269

7370

74-
def save_provenance(results_dir: str, parser: CoreParser) -> ProvPaths:
71+
def save_provenance(results_dir: str, parser: CoreParser, no_viewer: bool) -> ProvPaths:
7572
"""
7673
Store the provenance in results_dir.
7774
"""
@@ -86,6 +83,18 @@ def save_provenance(results_dir: str, parser: CoreParser) -> ProvPaths:
8683
"index_html_path": None,
8784
}
8885

86+
if no_viewer:
87+
paths.update(
88+
{
89+
"parameter_files_path": "N/A (No viewer generated)",
90+
"python_script_path": "N/A (No viewer generated)",
91+
"env_yml_path": "N/A (No viewer generated)",
92+
"index_html_path": "N/A (No viewer generated)",
93+
}
94+
)
95+
96+
return paths
97+
8998
paths["parameter_files_path"] = _save_parameter_files(prov_dir, parser)
9099
paths["python_script_path"] = _save_python_script(prov_dir, parser)
91100

@@ -409,9 +418,9 @@ def main(parameters=[]) -> List[CoreParameter]: # noqa B006
409418
if not os.path.exists(parameters[0].results_dir):
410419
os.makedirs(parameters[0].results_dir, 0o755)
411420

412-
if not parameters[0].no_viewer: # Only save provenance for full runs.
413-
prov_paths = save_provenance(parameters[0].results_dir, parser)
414-
421+
prov_paths = save_provenance(
422+
parameters[0].results_dir, parser, parameters[0].no_viewer
423+
)
415424
_log_diagnostic_run_info(prov_paths)
416425

417426
# Perform the diagnostic run
@@ -514,6 +523,7 @@ def _log_diagnostic_run_info(prov_paths: ProvPaths):
514523
env_yml_path,
515524
index_html_path,
516525
) = prov_paths.values()
526+
517527
logger.info(
518528
f"\n{'=' * 80}\n"
519529
f"E3SM Diagnostics Run\n"

0 commit comments

Comments
 (0)