diff --git a/e3sm_to_cmip/__main__.py b/e3sm_to_cmip/__main__.py index 5ecf3504..7276ebf9 100755 --- a/e3sm_to_cmip/__main__.py +++ b/e3sm_to_cmip/__main__.py @@ -2,17 +2,17 @@ A python command line tool to turn E3SM model output into CMIP6 compatable data. """ -from __future__ import absolute_import, division, print_function, unicode_literals - import argparse +import logging import os import signal +import subprocess import sys import tempfile import threading -import warnings from concurrent.futures import ProcessPoolExecutor as Pool from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from pprint import pprint from typing import List, Optional, Union @@ -22,7 +22,11 @@ from tqdm import tqdm from e3sm_to_cmip import ROOT_HANDLERS_DIR, __version__, resources -from e3sm_to_cmip._logger import _setup_logger, _setup_root_logger +from e3sm_to_cmip._logger import ( + _add_filehandler, + _setup_child_logger, + _setup_root_logger, +) from e3sm_to_cmip.cmor_handlers.utils import ( MPAS_REALMS, REALMS, @@ -42,18 +46,13 @@ find_mpas_files, get_handler_info_msg, precheck, - print_debug, - print_message, ) -os.environ["CDAT_ANONYMOUS_LOG"] = "false" - -warnings.filterwarnings("ignore") - +# Set up the root logger and module level logger. The module level logger is +# a child of the root logger. +_setup_root_logger() -# Setup the root logger and this module's logger. -log_filename = _setup_root_logger() -logger = _setup_logger(__name__, propagate=True) +logger = _setup_child_logger(__name__) @dataclass @@ -98,6 +97,9 @@ class CLIArguments: class E3SMtoCMIP: def __init__(self, args: Optional[List[str]] = None): + self.timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f") + self.log_filename = f"{self.timestamp}.log" + # A dictionary of command line arguments. parsed_args = self._parse_args(args) @@ -136,37 +138,100 @@ def __init__(self, args: Optional[List[str]] = None): self.cmor_log_dir: Optional[str] = parsed_args.logdir self.user_metadata: Optional[str] = parsed_args.user_metadata self.custom_metadata: Optional[str] = parsed_args.custom_metadata + + if self.output_path is not None: + self.output_path = os.path.abspath(self.output_path) + os.makedirs(self.output_path, exist_ok=True) + elif self.output_path is None: + self.output_path = os.path.join( + os.getcwd(), f"e3sm_to_cmip_run_{self.timestamp}" + ) + os.makedirs(self.output_path, exist_ok=True) + + # Setup directories using the CLI argument paths (e.g., output dir). + # ====================================================================== + self._setup_dirs_with_paths() + # Run the pre-check to determine if any of the variables have already # been CMORized. + # ====================================================================== if self.precheck_path is not None: self._run_precheck() + # Setup logger information and print out e3sm_to_cmip CLI arguments. + # ====================================================================== logger.info("--------------------------------------") logger.info("| E3SM to CMIP Configuration") logger.info("--------------------------------------") - logger.info(f" * var_list='{self.var_list}'") - logger.info(f" * input_path='{self.input_path}'") - logger.info(f" * output_path='{self.output_path}'") - logger.info(f" * precheck_path='{self.precheck_path}'") - logger.info(f" * freq='{self.freq}'") - logger.info(f" * realm='{self.realm}'") - logger.info(f" * Writing log output file to: {log_filename}") + config_details = { + "Timestamp": self.timestamp, + "Version Info": self._get_version_info(), + "Mode": ( + "Info" + if self.info_mode + else "Serial" + if self.serial_mode + else "Parallel" + ), + "Variable List": self.var_list, + "Input Path": self.input_path, + "Output Path": self.output_path, + "Precheck Path": self.precheck_path, + "Log Path": self.log_path, + "CMOR Log Path": self.cmor_log_dir, + "Temp Path for Processing MPAS Files": self.temp_path, + "Frequency": self.freq, + "Realm": self.realm, + } + + for key, value in config_details.items(): + logger.info(f" * {key}: {value}") + + # Load the CMOR handlers based on the realm and variable list. self.handlers = self._get_handlers() - def run(self): - # Setup logger information and print out e3sm_to_cmip CLI arguments. - # ====================================================================== - if self.output_path is not None: - self.new_metadata_path = os.path.join( - self.output_path, "user_metadata.json" + def _get_version_info(self) -> str: + """Retrieve version information for the current codebase. + + This method attempts to determine the current Git branch name and commit + hash of the repository containing this file. If the Git information + cannot be retrieved, it falls back to using the `__version__` variable. + + Returns + ------- + str + A string containing the Git branch name and commit hash in the + format "branch with commit ", or the + fallback version string in the format "version <__version__>" if Git + information is unavailable. + """ + try: + branch_name = ( + subprocess.check_output( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + cwd=os.path.dirname(__file__), + stderr=subprocess.DEVNULL, + ) + .strip() + .decode("utf-8") ) + commit_hash = ( + subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=os.path.dirname(__file__), + stderr=subprocess.DEVNULL, + ) + .strip() + .decode("utf-8") + ) + version_info = f"branch {branch_name} with commit {commit_hash}" + except subprocess.CalledProcessError: + version_info = f"version {__version__}" - # Setup directories using the CLI argument paths (e.g., output dir). - # ====================================================================== - if not self.info_mode: - self._setup_dirs_with_paths() + return version_info + def run(self): # Run e3sm_to_cmip with info mode. # ====================================================================== if self.info_mode: @@ -183,7 +248,7 @@ def run(self): status = self._run() if status != 0: - print_message( + logger.error( f"Error running handlers: { ' '.join([x['name'] for x in self.handlers]) }" ) return 1 @@ -497,8 +562,11 @@ def _setup_argparser(self) -> argparse.ArgumentParser: optional.add_argument( "--logdir", type=str, - default="./cmor_logs", - help="Where to put the logging output from CMOR.", + default="cmor_logs", + help=( + "The sub-directory that stores the CMOR logs. This sub-directory will " + "be stored under --output-path." + ), ) required_no_simple.add_argument( "-u", @@ -620,37 +688,54 @@ def _run_precheck(self): self.input_path, self.precheck_path, self.var_list, self.realm ) if not new_var_list: - print("All variables previously computed") + logger.info("All variables previously computed") if self.output_path is not None: os.mkdir(os.path.join(self.output_path, "CMIP6")) return 0 else: - print_message(f"Setting up conversion for {' '.join(new_var_list)}", "ok") + logger.info(f"Setting up conversion for {' '.join(new_var_list)}", "ok") self.var_list = new_var_list def _setup_dirs_with_paths(self): - # Create the output directory if it doesn't exist. - if not os.path.exists(self.output_path): # type: ignore - os.makedirs(self.output_path) # type: ignore + """Sets up various directories and paths required for e3sm_to_cmip. + + This method initializes paths for metadata, logs, and temporary storage. + It also updates the root logger's file path and copies user metadata + to the output directory if not in simple mode. + + Notes + ----- + If the environment variable `TMPDIR` is not set, a temporary directory + is created under the output path. + """ + self.new_metadata_path = os.path.join(self.output_path, "user_metadata.json") # type: ignore + self.cmor_log_dir = os.path.join(self.output_path, self.cmor_log_dir) # type: ignore + + # NOTE: Any warnings that appear before the log filehandler is + # instantiated will not be captured (e.g,. esmpy VersionWarning). + # However, they will still be captured by the console via a + # StreamHandler. + self.log_path = os.path.join(self.output_path, self.log_filename) # type: ignore + _add_filehandler(self.log_path) # Copy the user's metadata json file with the updated output directory - if not self.simple_mode: + if not self.simple_mode and not self.info_mode: copy_user_metadata(self.user_metadata, self.output_path) - # Setup temp storage directory - temp_path = os.environ.get("TMPDIR") - if temp_path is None: - temp_path = f"{self.output_path}/tmp" - if not os.path.exists(temp_path): - os.makedirs(temp_path) + if not self.info_mode: + self.temp_path = os.environ.get("TMPDIR") + + if self.temp_path is None: + self.temp_path = f"{self.output_path}/tmp" - tempfile.tempdir = temp_path + if not os.path.exists(self.temp_path): + os.makedirs(self.temp_path) - def _run_info_mode(self): # noqa: C901 - logger.info("--------------------------------------") - logger.info("| Running E3SM to CMIP in Info Mode") - logger.info("--------------------------------------") + tempfile.tempdir = self.temp_path + else: + self.temp_path = None + def _run_info_mode(self): # noqa: C901 messages = [] # if the user just asked for the handler info @@ -665,8 +750,11 @@ def _run_info_mode(self): # noqa: C901 for handler in self.handlers: table_info = _get_table_info(self.tables_path, handler["table"]) if handler["name"] not in table_info["variable_entry"]: - msg = f"Variable {handler['name']} is not included in the table {handler['table']}" - print_message(msg, status="error") + logger.error( + "Variable {handler['name']} is not included in the table " + f"{handler['table']}" + ) + continue else: if self.freq == "mon" and handler["table"] == "CMIP6_day.json": @@ -675,6 +763,7 @@ def _run_info_mode(self): # noqa: C901 "table" ] == "CMIP6_Amon.json": continue + hand_msg = get_handler_info_msg(handler) messages.append(hand_msg) @@ -684,6 +773,7 @@ def _run_info_mode(self): # noqa: C901 with xr.open_dataset(file_path) as ds: for handler in self.handlers: table_info = _get_table_info(self.tables_path, handler["table"]) + if handler["name"] not in table_info["variable_entry"]: continue @@ -694,8 +784,9 @@ def _run_info_mode(self): # noqa: C901 if raw_var not in ds.data_vars: has_vars = False - msg = f"Variable {handler['name']} is not present in the input dataset" - print_message(msg, status="error") + logger.error( + f"Variable {handler['name']} is not present in the input dataset" + ) break @@ -737,29 +828,26 @@ def _run_info_mode(self): # noqa: C901 with open(self.info_out_path, "w") as outstream: yaml.dump(messages, outstream) elif self.output_path is not None: - with open(self.output_path, "w") as outstream: + yaml_filepath = os.path.join(self.output_path, "info.yaml") + + with open(yaml_filepath, "w") as outstream: yaml.dump(messages, outstream) else: pprint(messages) def _run(self): if self.serial_mode: - mode_str = "Serial" run_func = self._run_serial else: - mode_str = "Parallel" run_func = self._run_parallel try: - logger.info("--------------------------------------") - logger.info(f"| Running E3SM to CMIP in {mode_str}") - logger.info("--------------------------------------") status = run_func() except KeyboardInterrupt: - print_message(" -- keyboard interrupt -- ", "error") + logger.error(" -- keyboard interrupt -- ") return 1 except Exception as e: - print_debug(e) + logger.error(e) return 1 return status @@ -830,7 +918,7 @@ def _run_serial(self) -> int: # noqa: C901 self.new_metadata_path, ) except Exception as e: - print_debug(e) + logger.error(e) if name is not None: num_success += 1 @@ -846,7 +934,7 @@ def _run_serial(self) -> int: # noqa: C901 pbar.close() except Exception as error: - print_debug(error) + logger.error(error) return 1 else: msg = f"{num_success} of {num_handlers} handlers complete" @@ -937,7 +1025,7 @@ def _run_parallel(self) -> int: # noqa: C901 logger.info(msg) except Exception as e: - print_debug(e) + logger.error(e) pbar.update(1) pbar.close() @@ -954,12 +1042,20 @@ def _run_parallel(self) -> int: # noqa: C901 return 0 def _timeout_exit(self): - print_message("Hit timeout limit, exiting") + logger.info("Hit timeout limit, exiting") os.kill(os.getpid(), signal.SIGINT) def main(args: Optional[List[str]] = None): app = E3SMtoCMIP(args) + + # Remove any existing filehandlers from the root logger. This prevents + # multiple filehandlers from being added to the root logger, which can + # cause log messages from newer runs to be written log files from older + # runs. + for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + app.run() diff --git a/e3sm_to_cmip/_logger.py b/e3sm_to_cmip/_logger.py index 2e14df7c..5f7c320c 100644 --- a/e3sm_to_cmip/_logger.py +++ b/e3sm_to_cmip/_logger.py @@ -1,68 +1,121 @@ +"""Logger module for setting up a custom logger.""" + import logging -import os -import time from datetime import datetime -from pytz import UTC +LOG_FORMAT = ( + "%(asctime)s [%(levelname)s]: %(filename)s(%(funcName)s:%(lineno)s) >> %(message)s" +) +LOG_FILEMODE = "w" +LOG_LEVEL = logging.INFO -def _setup_root_logger() -> str: # pragma: no cover - """Sets up the root logger. +class CustomFormatter(logging.Formatter): + def formatTime(self, record, datefmt=None): + # Includes microseconds up to 6 digits + dt = datetime.fromtimestamp(record.created) - The logger module will write to a log file and stream the console - simultaneously. + return dt.strftime("%Y-%m-%d %H:%M:%S.%f") - The log files are saved in a `/logs` directory relative to where - `e3sm_to_cmip` is executed. - Returns - ------- - str - The name of the logfile. +def _setup_root_logger(): + """Configures the root logger. + + This function sets up the root logger with a predefined format and log level. + It also enables capturing of warnings issued by the `warnings` module and + redirects them to the logging system. + + Notes + ----- + - The `force=True` parameter ensures that any existing logging configuration + is overridden. + - The file handler is added dynamically to the root logger later in the + E3SMtoCMIP class once the log file path is known. """ - os.makedirs("logs", exist_ok=True) - filename = f'logs/{UTC.localize(datetime.utcnow()).strftime("%Y%m%d_%H%M%S_%f")}' - log_format = "%(asctime)s_%(msecs)03d:%(levelname)s:%(funcName)s:%(message)s" + custom_formatter = CustomFormatter(LOG_FORMAT) + console_handler = logging.StreamHandler() + console_handler.setFormatter(custom_formatter) - # Setup the logging module. logging.basicConfig( - filename=filename, - format=log_format, - datefmt="%Y%m%d_%H%M%S", - level=logging.DEBUG, + level=LOG_LEVEL, + force=True, + handlers=[console_handler], ) - logging.captureWarnings(True) - logging.Formatter.converter = time.gmtime - - # Configure and add a console stream handler. - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.INFO) - log_formatter = logging.Formatter(log_format) - console_handler.setFormatter(log_formatter) - logging.getLogger().addHandler(console_handler) - return filename + logging.captureWarnings(True) -def _setup_logger(name, propagate=True) -> logging.Logger: - """Sets up a logger object. +def _setup_child_logger(name: str, propagate: bool = True) -> logging.Logger: + """Sets up a logger that is a child of the root logger. - This function is intended to be used at the top-level of a module. + This child logger inherits the root logger's handlers. Parameters ---------- name : str Name of the file where this function is called. propagate : bool, optional - Propogate this logger module's messages to the root logger or not, by - default True. + Whether to propagate logger messages or not, by default True. Returns ------- logging.Logger - The logger. + The child logger. + + Examples + --------- + Detailed information, typically of interest only when diagnosing problems: + + >>> logger.debug("") + + Confirmation that things are working as expected: + + >>> logger.info("") + + An indication that something unexpected happened, or indicative of some + problem in the near future: + + >>> logger.warning("") + + The software has not been able to perform some function due to a more + serious problem: + + >>> logger.error("") + + Similar to ``logger.error()``, but also outputs stack trace: + + >>> logger.exception("", exc_info=True) + + A serious error, indicating that the program itself may be unable to + continue running: + + >>> logger.critical("") """ logger = logging.getLogger(name) logger.propagate = propagate return logger + + +def _add_filehandler(log_path: str): + """Adds a file handler to the root logger dynamically. + + Adding the file handler will also create the log file automatically. + + Parameters + ---------- + log_path : str + The path to the log file. + + Notes + ----- + Any warnings that appear before the log filehandler is instantiated will not + be captured (e.g,. esmpy VersionWarning). However, they will still be + captured by the console via the default StreamHandler. + """ + file_handler = logging.FileHandler(log_path, mode=LOG_FILEMODE) + + custom_formatter = CustomFormatter(LOG_FORMAT) + file_handler.setFormatter(custom_formatter) + + logging.root.addHandler(file_handler) diff --git a/e3sm_to_cmip/cmor_handlers/handler.py b/e3sm_to_cmip/cmor_handlers/handler.py index c7e4510f..8d90bceb 100644 --- a/e3sm_to_cmip/cmor_handlers/handler.py +++ b/e3sm_to_cmip/cmor_handlers/handler.py @@ -11,11 +11,11 @@ import xcdat as xc import yaml -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _setup_child_logger from e3sm_to_cmip.cmor_handlers import FILL_VALUE, _formulas from e3sm_to_cmip.util import _get_table_for_non_monthly_freq -logger = _setup_logger(__name__) +logger = _setup_child_logger(__name__) # The names for valid hybrid sigma levels. HYBRID_SIGMA_LEVEL_NAMES = [ @@ -645,7 +645,10 @@ def _cmor_write( """ output_data = self._get_output_data(ds) - cmor.write(var_id=cmor_var_id, data=output_data) + try: + cmor.write(var_id=cmor_var_id, data=output_data) + except Exception as e: + logger.error(f"Error writing variable {self.name} to file: {e}") def _cmor_write_with_time( self, diff --git a/e3sm_to_cmip/cmor_handlers/utils.py b/e3sm_to_cmip/cmor_handlers/utils.py index 8f946e8e..b6f23f2e 100644 --- a/e3sm_to_cmip/cmor_handlers/utils.py +++ b/e3sm_to_cmip/cmor_handlers/utils.py @@ -11,11 +11,11 @@ LEGACY_HANDLER_DIR_PATH, MPAS_HANDLER_DIR_PATH, ) -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _setup_child_logger from e3sm_to_cmip.cmor_handlers.handler import VarHandler from e3sm_to_cmip.util import _get_table_for_non_monthly_freq -logger = _setup_logger(__name__) +logger = _setup_child_logger(__name__) # Type aliases Frequency = Literal["mon", "day", "6hrLev", "6hrPlev", "6hrPlevPt", "3hr", "1hr"] diff --git a/e3sm_to_cmip/cmor_handlers/vars/README.md b/e3sm_to_cmip/cmor_handlers/vars/README.md index e83aec11..39232d52 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/README.md +++ b/e3sm_to_cmip/cmor_handlers/vars/README.md @@ -6,4 +6,3 @@ to `handlers.yaml`. For example: - Some contain legacy `handle_simple()` functions that have since been refactored as a single `handle_simple()` function -- `phalf.py` and `pfull.py` still use `cdms2` and `cdutil` diff --git a/e3sm_to_cmip/cmor_handlers/vars/areacella.py b/e3sm_to_cmip/cmor_handlers/vars/areacella.py index 9f0d50b8..5014304d 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/areacella.py +++ b/e3sm_to_cmip/cmor_handlers/vars/areacella.py @@ -13,11 +13,11 @@ import xarray as xr from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _setup_child_logger from e3sm_to_cmip.mpas import write_netcdf from e3sm_to_cmip.util import print_message, setup_cmor -logger = _setup_logger(__name__) +logger = _setup_child_logger(__name__) # list of raw variable names needed RAW_VARIABLES = [str("area")] diff --git a/e3sm_to_cmip/cmor_handlers/vars/clisccp.py b/e3sm_to_cmip/cmor_handlers/vars/clisccp.py index 5402ef72..9ce9b0ea 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/clisccp.py +++ b/e3sm_to_cmip/cmor_handlers/vars/clisccp.py @@ -13,10 +13,10 @@ import numpy as np import xarray as xr -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _setup_child_logger from e3sm_to_cmip.util import print_message, setup_cmor -logger = _setup_logger(__name__) +logger = _setup_child_logger(__name__) # list of raw variable names needed RAW_VARIABLES = [str("FISCCP1_COSP")] diff --git a/e3sm_to_cmip/cmor_handlers/vars/orog.py b/e3sm_to_cmip/cmor_handlers/vars/orog.py index 7de757df..2ed2b167 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/orog.py +++ b/e3sm_to_cmip/cmor_handlers/vars/orog.py @@ -13,11 +13,11 @@ import xarray as xr from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _setup_child_logger from e3sm_to_cmip.mpas import write_netcdf from e3sm_to_cmip.util import print_message, setup_cmor -logger = _setup_logger(__name__) +logger = _setup_child_logger(__name__) # list of raw variable names needed RAW_VARIABLES = [str("PHIS")] diff --git a/e3sm_to_cmip/cmor_handlers/vars/sftlf.py b/e3sm_to_cmip/cmor_handlers/vars/sftlf.py index 127f7788..246ca2aa 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/sftlf.py +++ b/e3sm_to_cmip/cmor_handlers/vars/sftlf.py @@ -13,11 +13,11 @@ import xarray as xr from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _setup_child_logger from e3sm_to_cmip.mpas import write_netcdf from e3sm_to_cmip.util import print_message, setup_cmor -logger = _setup_logger(__name__) +logger = _setup_child_logger(__name__) # list of raw variable names needed RAW_VARIABLES = [str("LANDFRAC")] diff --git a/e3sm_to_cmip/mpas.py b/e3sm_to_cmip/mpas.py index b3f101fa..0fd77a29 100644 --- a/e3sm_to_cmip/mpas.py +++ b/e3sm_to_cmip/mpas.py @@ -728,7 +728,7 @@ def _compute_moc_time_series( lat_bnds = np.zeros((len(lat) - 1, 2)) lat_bnds[:, 0] = lat[0:-1] lat_bnds[:, 1] = lat[1:] - lat = 0.5 * (lat_bnds[:, 0] + lat_bnds[:, 1]) + lat = 0.5 * (lat_bnds[:, 0] + lat_bnds[:, 1]) # type: ignore lat_bnds = xarray.DataArray(lat_bnds, dims=("lat", "nbnd")) # type: ignore lat = xarray.DataArray(lat, dims=("lat",)) # type: ignore diff --git a/e3sm_to_cmip/util.py b/e3sm_to_cmip/util.py index 3627d9e5..3486c2a9 100644 --- a/e3sm_to_cmip/util.py +++ b/e3sm_to_cmip/util.py @@ -14,9 +14,9 @@ import yaml from tqdm import tqdm -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import _setup_child_logger -logger = _setup_logger(__name__) +logger = _setup_child_logger(__name__) ATMOS_TABLES = [ @@ -50,6 +50,7 @@ def print_debug(e): + # TODO: Deprecate this function. We use Python logger now. _, _, tb = sys.exc_info() traceback.print_tb(tb) print(e) @@ -73,11 +74,16 @@ def print_message(message, status="error"): """ Prints a message with either a green + or a red - + # TODO: Deprecate this function. We use Python logger now. Colors can't + # be captured in log files. + Parameters: message (str): the message to print - status (str): th""" + status (str): the status message. + """ + if status == "error": - print( + logger.error( colors.FAIL + "[-] " + colors.ENDC @@ -86,11 +92,11 @@ def print_message(message, status="error"): + colors.ENDC ) elif status == "ok": - print(colors.OKGREEN + "[+] " + colors.ENDC + str(message)) + logger.info(colors.OKGREEN + "[+] " + colors.ENDC + str(message)) elif status == "info": - print(str(message)) + logger.info(str(message)) elif status == "debug": - print( + logger.info( colors.OKBLUE + "[*] " + colors.ENDC diff --git a/tests/run_script.py b/tests/run_script.py index 99d262f0..50134f94 100644 --- a/tests/run_script.py +++ b/tests/run_script.py @@ -9,20 +9,25 @@ NOTE: This script can only be executed on LCRC machines. """ -import datetime +import os from e3sm_to_cmip.__main__ import main # The list of variables to process. Update as needed. -VAR_LIST = ( - "pfull, phalf, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, " - "tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, cl, clw, cli, clivi, clwvi, prw, " - "rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer " - "rsdscs, hur" -) +# VAR_LIST = ( +# "pfull, phalf, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, " +# "tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, cl, clw, cli, clivi, clwvi, prw, " +# "rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer " +# "rsdscs, hur" +# )1 + +VAR_LIST = "pfull, phalf, tas" # The output path for CMORized datasets. Update as needed. -OUTPUT_PATH = f"../qa/run_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}" +OUTPUT_PATH = ( + "/lcrc/group/e3sm/public_html/e3sm_to_cmip/feature-274-redesign-logger-info" +) + args = [ "--var-list", @@ -35,9 +40,17 @@ "/lcrc/group/e3sm/e3sm_to_cmip/cmip6-cmor-tables/Tables/", "--user-metadata", "/lcrc/group/e3sm/e3sm_to_cmip/CMIP6-Metadata/template.json", - "--serial", + "--info", ] # `main()` creates an `E3SMtoCMIP` object and passes `args` to it, which sets # the object parameters to execute a run. main(args) + +# Ensure the path and its contents have the correct permissions recursively +for root, dirs, files in os.walk(OUTPUT_PATH): + os.chmod(root, 0o505) # o=rx (read and execute for others) + for d in dirs: + os.chmod(os.path.join(root, d), 0o505) + for f in files: + os.chmod(os.path.join(root, f), 0o404) # o=r (read for others)