Skip to content

Commit dcd5719

Browse files
tomvothecodertomvothecoder
authored
Fix prov files not printing with links in prov index.html (#937)
* Consolidate logger messages for prov paths * Suppress error for git command when the package is a version build
1 parent 4e35711 commit dcd5719

File tree

2 files changed

+201
-138
lines changed

2 files changed

+201
-138
lines changed

‎e3sm_diags/e3sm_diags_driver.py‎

Lines changed: 201 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,20 @@
11
#!/usr/bin/env python
22
# The above line is needed for `test_all_sets.test_all_sets_mpl`.
33
# Otherwise, OSError: [Errno 8] Exec format error: 'e3sm_diags_driver.py'.
4+
from __future__ import annotations
5+
46
import os
57
import subprocess
68
import sys
79
import traceback
8-
from typing import Dict, List, Tuple
10+
from datetime import datetime
11+
from typing import Dict, List, Tuple, TypedDict
912

1013
import dask
1114
import dask.bag as db
1215

1316
import e3sm_diags
14-
from e3sm_diags.logger import custom_logger
17+
from e3sm_diags.logger import LOG_FILENAME, custom_logger
1518
from e3sm_diags.parameter.core_parameter import CoreParameter
1619
from e3sm_diags.parser import SET_TO_PARSER
1720
from e3sm_diags.parser.core_parser import CoreParser
@@ -20,6 +23,34 @@
2023
logger = custom_logger(__name__)
2124

2225

26+
class ProvPaths(TypedDict):
27+
"""
28+
ProvPaths is a TypedDict that defines the structure for provenance paths.
29+
30+
Attributes
31+
----------
32+
results_dir: str
33+
Path to the diagnostic results.
34+
log_path : str
35+
Path to the log directory.
36+
parameter_files_path : str
37+
Path to the parameter files.
38+
python_script_path : str
39+
Path to the Python script.
40+
env_yml_path : str
41+
Path to the environment YAML file.
42+
index_html_path : str
43+
Path to the provenance index HTML file.
44+
"""
45+
46+
results_dir: str
47+
log_path: str
48+
parameter_files_path: str | None
49+
python_script_path: str | None
50+
env_yml_path: str | None
51+
index_html_path: str | None
52+
53+
2354
def get_default_diags_path(set_name, run_type, print_path=True):
2455
"""
2556
Returns the path for the default diags for plotset set_name.
@@ -40,132 +71,146 @@ def get_default_diags_path(set_name, run_type, print_path=True):
4071
return pth
4172

4273

43-
def _save_env_yml(results_dir):
74+
def save_provenance(results_dir: str, parser: CoreParser) -> ProvPaths:
75+
"""
76+
Store the provenance in results_dir.
77+
"""
78+
prov_dir = os.path.join(results_dir, "prov")
79+
80+
paths: ProvPaths = {
81+
"results_dir": results_dir,
82+
"log_path": os.path.join(prov_dir, LOG_FILENAME),
83+
"parameter_files_path": None,
84+
"python_script_path": None,
85+
"env_yml_path": None,
86+
"index_html_path": None,
87+
}
88+
89+
paths["parameter_files_path"] = _save_parameter_files(prov_dir, parser)
90+
paths["python_script_path"] = _save_python_script(prov_dir, parser)
91+
92+
# FIXME: Replace Exception with specific exception type.
93+
try:
94+
paths["env_yml_path"] = _save_env_yml(prov_dir)
95+
except Exception:
96+
paths["env_yml_path"] = None
97+
traceback.print_exc()
98+
99+
if not os.path.exists(prov_dir):
100+
os.makedirs(prov_dir, 0o755)
101+
102+
# Create an HTML file to list the contents of the prov dir.
103+
index_html_path = os.path.join(prov_dir, "index.html")
104+
paths["index_html_path"] = index_html_path
105+
106+
with open(index_html_path, "w") as f:
107+
f.write("<html><body><h1>Provenance Files</h1><ul>")
108+
109+
for root, _, files in os.walk(prov_dir):
110+
for file_name in files:
111+
file_path = os.path.relpath(os.path.join(root, file_name), prov_dir)
112+
f.write(
113+
f'<li><a href="{file_path}" target="_blank">{file_name}</a></li>'
114+
)
115+
116+
f.write("</ul></body></html>")
117+
118+
return paths
119+
120+
121+
def _save_env_yml(results_dir: str) -> str | None:
44122
"""
45123
Save the yml to recreate the environment in results_dir.
46124
"""
47125
cmd = "conda env export"
48126
p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
49127
output, err = p.communicate()
50128

129+
filename = None
130+
51131
if err:
52132
logger.exception("Error when creating env yml file: ")
53133
logger.exception(err)
54134
else:
55-
fnm = os.path.join(results_dir, "environment.yml")
56-
with open(fnm, "w") as f:
135+
filename = os.path.join(results_dir, "environment.yml")
136+
137+
with open(filename, "w") as f:
57138
f.write(output.decode("utf-8"))
58-
logger.info("Saved environment yml file to: {}".format(fnm))
59139

140+
return filename
60141

61-
def _save_parameter_files(results_dir, parser):
142+
143+
def _save_parameter_files(results_dir: str, parser: CoreParser) -> str | None:
62144
"""
63145
Save the command line arguments used, and any py or cfg files.
64146
"""
147+
filepath = os.path.join(results_dir, "cmd_used.txt")
148+
new_filepath = None
149+
65150
cmd_used = " ".join(sys.argv)
66-
fnm = os.path.join(results_dir, "cmd_used.txt")
67-
with open(fnm, "w") as f:
151+
with open(filepath, "w") as f:
68152
f.write(cmd_used)
69-
logger.info("Saved command used to: {}".format(fnm))
70153

71154
args = parser.view_args()
72155

73156
if hasattr(args, "parameters") and args.parameters:
74-
fnm = args.parameters
75-
if not os.path.isfile(fnm):
76-
logger.warning("File does not exist: {}".format(fnm))
77-
else:
78-
with open(fnm, "r") as f:
79-
contents = "".join(f.readlines())
80-
# Remove any path, just keep the filename.
81-
new_fnm = fnm.split("/")[-1]
82-
new_fnm = os.path.join(results_dir, new_fnm)
83-
with open(new_fnm, "w") as f:
84-
f.write(contents)
85-
logger.info("Saved py file to: {}".format(new_fnm))
86-
87-
if hasattr(args, "other_parameters") and args.other_parameters:
88-
fnm = args.other_parameters[0]
89-
if not os.path.isfile(fnm):
90-
logger.warning("File does not exist: {}".format(fnm))
91-
else:
92-
with open(fnm, "r") as f:
93-
contents = "".join(f.readlines())
94-
# Remove any path, just keep the filename.
95-
new_fnm = fnm.split("/")[-1]
96-
new_fnm = os.path.join(results_dir, new_fnm)
97-
with open(new_fnm, "w") as f:
98-
f.write(contents)
99-
logger.info("Saved cfg file to: {}".format(new_fnm))
157+
filepath = args.parameters
158+
elif hasattr(args, "other_parameters") and args.other_parameters:
159+
filepath = args.other_parameters[0]
160+
161+
if not os.path.isfile(filepath):
162+
logger.warning("File does not exist: {}".format(filepath))
163+
else:
164+
with open(filepath, "r") as f:
165+
contents = "".join(f.readlines())
166+
167+
# Remove any path, just keep the filename.
168+
new_filepath = filepath.split("/")[-1]
169+
new_filepath = os.path.join(results_dir, new_filepath)
170+
171+
with open(new_filepath, "w") as f:
172+
f.write(contents)
100173

174+
return new_filepath
101175

102-
def _save_python_script(results_dir, parser):
176+
177+
def _save_python_script(results_dir: str, parser: CoreParser) -> str | None:
103178
"""
104179
When using a Python script to run the
105180
diags via the API, dump a copy of the script.
106181
"""
107182
args = parser.view_args()
108-
# If running the legacy way, there's
109-
# nothing to be saved.
183+
184+
# FIXME: Is this code still needed?
185+
# If running the legacy way, there's nothing to be saved.
110186
if args.parameters:
111-
return
187+
return None
112188

113189
# Get the last argument that has .py in it.
114190
py_files = [f for f in sys.argv if f.endswith(".py")]
191+
115192
# User didn't pass in a Python file, so they maybe ran:
116193
# e3sm_diags -d diags.cfg
117194
if not py_files:
118-
return
195+
return None
119196

120197
fnm = py_files[-1]
121198

122199
if not os.path.isfile(fnm):
123200
logger.warning("File does not exist: {}".format(fnm))
124-
return
201+
return None
125202

126203
with open(fnm, "r") as f:
127204
contents = "".join(f.readlines())
128-
# Remove any path, just keep the filename.
129-
new_fnm = fnm.split("/")[-1]
130-
new_fnm = os.path.join(results_dir, new_fnm)
131-
with open(new_fnm, "w") as f:
132-
f.write(contents)
133-
logger.info("Saved Python script to: {}".format(new_fnm))
134-
135-
136-
def save_provenance(results_dir, parser):
137-
"""
138-
Store the provenance in results_dir.
139-
"""
140-
results_dir = os.path.join(results_dir, "prov")
141-
if not os.path.exists(results_dir):
142-
os.makedirs(results_dir, 0o755)
143-
144-
# Create an HTML file to list the contents of the prov dir.
145-
index_html_path = os.path.join(results_dir, "index.html")
146-
147-
with open(index_html_path, "w") as f:
148-
f.write("<html><body><h1>Provenance Files</h1><ul>")
149-
150-
for file_name in os.listdir(results_dir):
151-
file_path = os.path.join(results_dir, file_name)
152-
if os.path.isfile(file_path):
153-
f.write(
154-
f'<li><a href="{file_name}" target="_blank">{file_name}</a></li>'
155-
)
156205

157-
f.write("</ul></body></html>")
158-
159-
logger.info("Created provenance index HTML file at: {}".format(index_html_path))
160-
161-
try:
162-
_save_env_yml(results_dir)
163-
except Exception:
164-
traceback.print_exc()
206+
# Remove any path, just keep the filename.
207+
new_filepath = fnm.split("/")[-1]
208+
new_filepath = os.path.join(results_dir, new_filepath)
165209

166-
_save_parameter_files(results_dir, parser)
210+
with open(new_filepath, "w") as f:
211+
f.write(contents)
167212

168-
_save_python_script(results_dir, parser)
213+
return new_filepath
169214

170215

171216
# FIXME: B008 Do not perform function call `CoreParser` in argument defaults;
@@ -363,8 +408,11 @@ def main(parameters=[]) -> List[CoreParameter]: # noqa B006
363408

364409
if not os.path.exists(parameters[0].results_dir):
365410
os.makedirs(parameters[0].results_dir, 0o755)
411+
366412
if not parameters[0].no_viewer: # Only save provenance for full runs.
367-
save_provenance(parameters[0].results_dir, parser)
413+
prov_paths = save_provenance(parameters[0].results_dir, parser)
414+
415+
_log_diagnostic_run_info(prov_paths)
368416

369417
# Perform the diagnostic run
370418
# --------------------------
@@ -410,5 +458,77 @@ def main(parameters=[]) -> List[CoreParameter]: # noqa B006
410458
return parameters_results
411459

412460

461+
def _log_diagnostic_run_info(prov_paths: ProvPaths):
462+
"""Logs information about the diagnostic run.
463+
464+
This method is useful for tracking the provenance of the diagnostic run
465+
and understanding the context of the diagnostic results.
466+
467+
It logs the following information:
468+
- Timestamp of the run
469+
- Version information (Git branch and commit hash or module version)
470+
- Paths to the provenance files (log, parameter files, Python script,
471+
env yml, index HTML)
472+
473+
Parameters
474+
----------
475+
prov_paths : ProvPaths
476+
The paths to the provenance files.
477+
478+
Notes
479+
-----
480+
The version information is retrieved from the current Git branch and
481+
commit hash. If the Git information is not available, it falls back
482+
to the version defined in the `e3sm_diags` module.
483+
"""
484+
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
485+
486+
try:
487+
branch_name = (
488+
subprocess.check_output(
489+
["git", "rev-parse", "--abbrev-ref", "HEAD"],
490+
cwd=os.path.dirname(__file__),
491+
stderr=subprocess.DEVNULL,
492+
)
493+
.strip()
494+
.decode("utf-8")
495+
)
496+
commit_hash = (
497+
subprocess.check_output(
498+
["git", "rev-parse", "HEAD"],
499+
cwd=os.path.dirname(__file__),
500+
stderr=subprocess.DEVNULL,
501+
)
502+
.strip()
503+
.decode("utf-8")
504+
)
505+
version_info = f"branch {branch_name} with commit {commit_hash}"
506+
except subprocess.CalledProcessError:
507+
version_info = f"version {e3sm_diags.__version__}"
508+
509+
(
510+
results_dir,
511+
log_path,
512+
parameter_files_path,
513+
python_script_path,
514+
env_yml_path,
515+
index_html_path,
516+
) = prov_paths.values()
517+
logger.info(
518+
f"\n{'=' * 80}\n"
519+
f"E3SM Diagnostics Run\n"
520+
f"{'-' * 20}\n"
521+
f"Timestamp: {timestamp}\n"
522+
f"Version Info: {version_info}\n"
523+
f"Results Path: {results_dir}\n"
524+
f"Log Path: {log_path}\n"
525+
f"Parameter Files Path: {parameter_files_path}\n"
526+
f"Python Script Path: {python_script_path}\n"
527+
f"Environment YML Path: {env_yml_path}\n"
528+
f"Provenance Index HTML Path: {index_html_path}\n"
529+
f"{'=' * 80}\n"
530+
)
531+
532+
413533
if __name__ == "__main__":
414534
main()

0 commit comments

Comments
 (0)