From fdbe225c1bf7e5c714ecd310181f6201de31d918 Mon Sep 17 00:00:00 2001 From: Tony Bartoletti Date: Thu, 27 Jun 2024 13:37:26 -0700 Subject: [PATCH 1/9] consolidate cmor setups and e2c logging --- e3sm_to_cmip/__main__.py | 107 +++++++++------ e3sm_to_cmip/_logger.py | 122 +++++++++--------- e3sm_to_cmip/cmor_handlers/handler.py | 37 +++--- .../cmor_handlers/mpas_vars/areacello.py | 12 +- .../cmor_handlers/mpas_vars/fsitherm.py | 10 +- e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py | 9 +- .../cmor_handlers/mpas_vars/hfsifrazil.py | 9 +- .../cmor_handlers/mpas_vars/masscello.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py | 9 +- .../cmor_handlers/mpas_vars/mlotst.py | 9 +- .../cmor_handlers/mpas_vars/msftmz.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py | 9 +- .../cmor_handlers/mpas_vars/siconc.py | 9 +- .../cmor_handlers/mpas_vars/simass.py | 9 +- .../cmor_handlers/mpas_vars/sisnmass.py | 9 +- .../cmor_handlers/mpas_vars/sisnthick.py | 9 +- .../cmor_handlers/mpas_vars/sitemptop.py | 10 +- .../cmor_handlers/mpas_vars/sithick.py | 9 +- .../cmor_handlers/mpas_vars/sitimefrac.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py | 11 +- e3sm_to_cmip/cmor_handlers/mpas_vars/so.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py | 9 +- .../cmor_handlers/mpas_vars/thetao.py | 11 +- .../cmor_handlers/mpas_vars/thetaoga.py | 9 +- .../cmor_handlers/mpas_vars/thkcello.py | 20 ++- e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py | 9 +- .../cmor_handlers/mpas_vars/volcello.py | 12 +- e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py | 9 +- e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py | 9 +- .../cmor_handlers/mpas_vars/zhalfo.py | 12 +- e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py | 9 +- e3sm_to_cmip/cmor_handlers/utils.py | 38 ++---- e3sm_to_cmip/cmor_handlers/vars/areacella.py | 51 +++----- e3sm_to_cmip/cmor_handlers/vars/clisccp.py | 38 ++---- e3sm_to_cmip/cmor_handlers/vars/orog.py | 47 ++----- e3sm_to_cmip/cmor_handlers/vars/sftlf.py | 47 ++----- e3sm_to_cmip/mpas.py | 59 ++++----- e3sm_to_cmip/util.py | 27 ++-- 51 files changed, 479 insertions(+), 489 deletions(-) diff --git a/e3sm_to_cmip/__main__.py b/e3sm_to_cmip/__main__.py index c939abb9..973de7b0 100755 --- a/e3sm_to_cmip/__main__.py +++ b/e3sm_to_cmip/__main__.py @@ -21,8 +21,12 @@ from tqdm import tqdm from e3sm_to_cmip import ROOT_HANDLERS_DIR, __version__, resources -from e3sm_to_cmip._logger import _setup_logger, _setup_root_logger +from datetime import datetime, timezone + +from e3sm_to_cmip._logger import e2c_logger + from e3sm_to_cmip.cmor_handlers.utils import ( + instantiate_h_utils_logger, MPAS_REALMS, REALMS, Frequency, @@ -32,7 +36,10 @@ derive_handlers, load_all_handlers, ) +from e3sm_to_cmip.cmor_handlers.handler import instantiate_handler_logger + from e3sm_to_cmip.util import ( + instantiate_util_logger, FREQUENCIES, _get_table_info, get_handler_info_msg, @@ -50,11 +57,6 @@ warnings.filterwarnings("ignore") -# Setup the root logger and this module's logger. -log_filename = _setup_root_logger() -logger = _setup_logger(__name__, propagate=True) - - @dataclass class CLIArguments: """A data class storing the command line arguments for e3sm_to_cmip. @@ -97,9 +99,14 @@ class CLIArguments: class E3SMtoCMIP: def __init__(self, args: Optional[List[str]] = None): + global logger # A dictionary of command line arguments. parsed_args = self._parse_args(args) + # Setup this module's logger AFTER args parsed in E3SMtoCMIP init, so that + # default log file is NOT created for mere "--help" or "--version" calls. + logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True) + # NOTE: The order of these attributes align with class CLIArguments. # ====================================================================== # Run Mode settings. @@ -115,8 +122,6 @@ def __init__(self, args: Optional[List[str]] = None): self.debug: bool = parsed_args.debug self.timeout: int = parsed_args.timeout - # ====================================================================== - # CMOR settings. # ====================================================================== self.var_list: List[str] = self._get_var_list(parsed_args.var_list) self.realm: Union[Realm, MPASRealm] = parsed_args.realm @@ -140,6 +145,9 @@ def __init__(self, args: Optional[List[str]] = None): if self.precheck_path is not None: self._run_precheck() + self.handlers = self._get_handlers() + + def echo_settings(self): logger.info("--------------------------------------") logger.info("| E3SM to CMIP Configuration") logger.info("--------------------------------------") @@ -149,9 +157,7 @@ def __init__(self, args: Optional[List[str]] = None): logger.info(f" * precheck_path='{self.precheck_path}'") logger.info(f" * freq='{self.freq}'") logger.info(f" * realm='{self.realm}'") - logger.info(f" * Writing log output file to: {log_filename}") - - self.handlers = self._get_handlers() + # logger.info(f" * Writing log output file to: {log_filename}") def run(self): # Setup logger information and print out e3sm_to_cmip CLI arguments. @@ -161,17 +167,16 @@ def run(self): self.output_path, "user_metadata.json" ) - # Setup directories using the CLI argument paths (e.g., output dir). - # ====================================================================== - if not self.info_mode: - self._setup_dirs_with_paths() - # Run e3sm_to_cmip with info mode. # ====================================================================== if self.info_mode: self._run_info_mode() sys.exit(0) + # Setup directories using the CLI argument paths (e.g., output dir). + # ====================================================================== + self._setup_dirs_with_paths() + # Run e3sm_to_cmip to CMORize serially or in parallel. # ====================================================================== timer = None @@ -306,7 +311,8 @@ def _parse_args(self, args: Optional[List[str]]) -> CLIArguments: sys.exit(1) # Parse the arguments and perform validation. - parsed_args = argparser.parse_args(args_to_parse) + parsed_args = argparser.parse_args(args_to_parse) # (exits here if args == "-h" or "--help" or "--version") + self._validate_parsed_args(parsed_args) # Convert to this data class for type checking to work. @@ -531,7 +537,7 @@ def _setup_argparser(self) -> argparse.ArgumentParser: "-h", "--help", action="help", - default=argparse.SUPPRESS, + # default=argparse.SUPPRESS, help="show this help message and exit", ) @@ -635,13 +641,13 @@ def _setup_dirs_with_paths(self): copy_user_metadata(self.user_metadata, self.output_path) # Setup temp storage directory - temp_path = os.environ.get("TMPDIR") - if temp_path is None: - temp_path = f"{self.output_path}/tmp" - if not os.path.exists(temp_path): - os.makedirs(temp_path) + # temp_path = os.environ.get("TMPDIR") + # if temp_path is None: + # temp_path = f"{self.output_path}/tmp" + # if not os.path.exists(temp_path): + # os.makedirs(temp_path) - tempfile.tempdir = temp_path + # tempfile.tempdir = temp_path def _run_info_mode(self): # noqa: C901 logger.info("--------------------------------------") @@ -778,12 +784,15 @@ def _run_serial(self) -> int: # noqa: C901 int 1 if an error occurs, else 0 """ + + do_pbar = False try: num_handlers = len(self.handlers) num_success = 0 + num_failure = 0 name = None - if self.realm != "atm": + if self.realm != "atm" and do_pbar: pbar = tqdm(total=len(self.handlers)) for _, handler in enumerate(self.handlers): @@ -816,7 +825,7 @@ def _run_serial(self) -> int: # noqa: C901 } msg = f"Trying to CMORize with handler: {handler}" - logger.info(msg) + logger.critical(msg) # NOTE: We need a try and except statement here for TypeError because # the VarHandler.cmorize method does not use **kwargs, while the handle @@ -838,17 +847,18 @@ def _run_serial(self) -> int: # noqa: C901 except Exception as e: print_debug(e) - if name is not None: + if name is not None and name is not "": num_success += 1 - msg = f"Finished {name}, {num_success}/{num_handlers} jobs complete" + msg = f"Finished {name}, {num_success}/{num_handlers} jobs complete (via run_serial)" logger.info(msg) else: + num_failure += 1 msg = f"Error running handler {handler['name']}" logger.info(msg) - if self.realm != "atm": + if self.realm != "atm" and do_pbar: pbar.update(1) - if self.realm != "atm": + if self.realm != "atm" and do_pbar: pbar.close() except Exception as error: @@ -858,6 +868,8 @@ def _run_serial(self) -> int: # noqa: C901 msg = f"{num_success} of {num_handlers} handlers complete" logger.info(msg) + if num_failure > 0: + return 1 return 0 def _run_parallel(self) -> int: # noqa: C901 @@ -872,6 +884,8 @@ def _run_parallel(self) -> int: # noqa: C901 pool_res = list() will_run = [] + do_pbar = False + for idx, handler in enumerate(self.handlers): handler_method = handler["method"] handler_variables = handler["raw_variables"] @@ -926,8 +940,10 @@ def _run_parallel(self) -> int: # noqa: C901 pool_res.append(res) # wait for each result to complete - pbar = tqdm(total=len(pool_res)) + if do_pbar: + pbar = tqdm(total=len(pool_res)) num_success = 0 + num_failure = 0 num_handlers = len(self.handlers) finished_success = [] for idx, res in enumerate(pool_res): @@ -936,26 +952,30 @@ def _run_parallel(self) -> int: # noqa: C901 finished_success.append(out) if out: num_success += 1 - msg = f"Finished {out}, {idx + 1}/{num_handlers} jobs complete" + msg = f"Finished {out}, {idx + 1}/{num_handlers} jobs complete (via run_parallel)" else: + num_failure += 1 msg = f'Error running handler {self.handlers[idx]["name"]}' logger.error(msg) logger.info(msg) except Exception as e: print_debug(e) - pbar.update(1) + if do_pbar: + pbar.update(1) - pbar.close() + if do_pbar: + pbar.close() pool.shutdown() msg = f"{num_success} of {num_handlers} handlers complete" logger.info(msg) failed = set(will_run) - set(finished_success) - if failed: + if failed or num_failure > 0: logger.error(f"{', '.join(list(failed))} failed to complete") logger.error(msg) + return 1 return 0 @@ -963,11 +983,26 @@ def _timeout_exit(self): print_message("Hit timeout limit, exiting") os.kill(os.getpid(), signal.SIGINT) +logger=None def main(args: Optional[List[str]] = None): + global logger +def main(args: Optional[List[str]] = None): + global logger + app = E3SMtoCMIP(args) - app.run() + + # These calls allow loggers that create default logfiles to avoid being + # instantiated by arguments "--help" or "--version". + instantiate_util_logger() + instantiate_h_utils_logger() + instantiate_handler_logger() + + app.echo_settings() + return app.run() if __name__ == "__main__": + + main() diff --git a/e3sm_to_cmip/_logger.py b/e3sm_to_cmip/_logger.py index 2e14df7c..bcf08ecf 100644 --- a/e3sm_to_cmip/_logger.py +++ b/e3sm_to_cmip/_logger.py @@ -1,68 +1,66 @@ +import inspect import logging import os -import time -from datetime import datetime - -from pytz import UTC - - -def _setup_root_logger() -> str: # pragma: no cover - """Sets up the root logger. - - The logger module will write to a log file and stream the console - simultaneously. - - The log files are saved in a `/logs` directory relative to where - `e3sm_to_cmip` is executed. - - Returns - ------- - str - The name of the logfile. - """ - os.makedirs("logs", exist_ok=True) - filename = f'logs/{UTC.localize(datetime.utcnow()).strftime("%Y%m%d_%H%M%S_%f")}' - log_format = "%(asctime)s_%(msecs)03d:%(levelname)s:%(funcName)s:%(message)s" - - # Setup the logging module. - logging.basicConfig( - filename=filename, - format=log_format, - datefmt="%Y%m%d_%H%M%S", - level=logging.DEBUG, - ) - logging.captureWarnings(True) - logging.Formatter.converter = time.gmtime - - # Configure and add a console stream handler. - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.INFO) - log_formatter = logging.Formatter(log_format) - console_handler.setFormatter(log_formatter) - logging.getLogger().addHandler(console_handler) - - return filename - - -def _setup_logger(name, propagate=True) -> logging.Logger: - """Sets up a logger object. - - This function is intended to be used at the top-level of a module. - - Parameters - ---------- - name : str - Name of the file where this function is called. - propagate : bool, optional - Propogate this logger module's messages to the root logger or not, by - default True. - - Returns - ------- - logging.Logger - The logger. - """ +from datetime import datetime, timezone + +''' + ACCEPTS: + name= + logfilename= [default = e2c_logs/dflt_log-{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S_%f')}.log] + log_level= [default = DEBUG] + to_console=[True|False] [default = False] + to_logfile=[True|False] [default = False] + propagate=[True|False] [default = False] +''' + +default_log_dir = "e2c_logs" +default_log = f"{default_log_dir}/e2c_root_log-{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S_%f')}.log" + +def e2c_logger(name=None, logfilename=default_log, set_log_level=None, to_console=False, to_logfile=False, propagate=False): + + # print(f"DEBUG: _logger: entered e2c_logger at {datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S_%f')} called by {name}", flush=True) + + default_log_lvl = logging.DEBUG + + # create logging directory as required + + if to_logfile: + dn = os.path.dirname(logfilename) + if len(dn) and not os.path.exists(dn): + os.makedirs(dn) + logger = logging.getLogger(name) + if name == None or name == "__main__": + logger = logger.root + logger.propagate = propagate + if set_log_level == "None" or set_log_level == "DEBUG": + log_level = default_log_lvl + elif set_log_level == "INFO": + log_level = logging.INFO + elif set_log_level == "WARNING": + log_level = logging.WARNING + elif set_log_level == "ERROR": + log_level = logging.ERROR + elif set_log_level == "CRITICAL": + log_level = logging.CRITICAL + else: log_level = default_log_lvl + + logger.setLevel(log_level) + + logger.handlers = [] + + if to_console: + logStreamHandler = logging.StreamHandler() + logStreamHandler.setFormatter(logging.Formatter("%(asctime)s_%(msecs)03d:%(levelname)s:%(name)s:%(funcName)s:%(message)s",datefmt="%Y%m%d_%H%M%S")) + logger.addHandler(logStreamHandler) + + if to_logfile: + logFileHandler = logging.FileHandler(logfilename) + logFileHandler.setFormatter(logging.Formatter("%(asctime)s_%(msecs)03d:%(levelname)s:%(name)s:%(funcName)s:%(message)s",datefmt="%Y%m%d_%H%M%S")) + logger.addHandler(logFileHandler) + return logger + + diff --git a/e3sm_to_cmip/cmor_handlers/handler.py b/e3sm_to_cmip/cmor_handlers/handler.py index 7766504e..655df5c1 100644 --- a/e3sm_to_cmip/cmor_handlers/handler.py +++ b/e3sm_to_cmip/cmor_handlers/handler.py @@ -2,7 +2,6 @@ import abc import json -import logging import os from typing import Any, Dict, KeysView, List, Literal, Optional, Tuple, TypedDict @@ -12,11 +11,17 @@ import xcdat as xc import yaml -from e3sm_to_cmip._logger import _setup_logger from e3sm_to_cmip.cmor_handlers import FILL_VALUE, _formulas from e3sm_to_cmip.util import _get_table_for_non_monthly_freq +from e3sm_to_cmip._logger import e2c_logger + +logger=None + +def instantiate_handler_logger(): + global logger + + logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=True) -logger = _setup_logger(__name__) # The names for valid hybrid sigma levels. HYBRID_SIGMA_LEVEL_NAMES = [ @@ -267,7 +272,7 @@ def cmorize( # the variables. Otherwise, the IDs of cmor objects gets wiped after # every loop. cmor.close() - logger.debug( + logger.info( f"{self.name}: CMORized and file write complete, closing CMOR I/O." ) @@ -290,7 +295,7 @@ def _all_vars_have_filepaths( """ for var, filepaths in vars_to_filespaths.items(): if len(filepaths) == 0: - logging.error(f"{var}: Unable to find input files for {var}") + logger.error(f"{var}: Unable to find input files for {var}") return False return True @@ -317,22 +322,22 @@ def _setup_cmor_module( logdir : str | None The optional log directory. """ - if log_dir is not None: - logpath = log_dir - else: - cwd = os.getcwd() - logpath = os.path.join(cwd, "cmor_logs") + # if log_dir is not None: + # logpath = log_dir + # else: + # cwd = os.getcwd() + # logpath = os.path.join(cwd, "cmor_logs") - os.makedirs(logpath, exist_ok=True) - logfile = os.path.join(logpath, var_name + ".log") + # os.makedirs(logpath, exist_ok=True) + # logfile = os.path.join(logpath, var_name + ".log") - cmor.setup( - inpath=tables_path, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile - ) + print("DEBUG: EMPLOYED HANDLER.PY _setup_cmor_module()", flush=True) + + cmor.setup(inpath=tables_path, netcdf_file_action=cmor.CMOR_REPLACE,logfile="/dev/null") cmor.dataset_json(metadata_path) cmor.load_table(self.table) - logging.info(f"{var_name}: CMOR setup complete") + logger.info(f"{var_name}: CMOR setup complete") def _get_var_time_dim(self, table_path: str) -> str | None: """Get the CMIP variable's time dimension, if it exists. diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py index 867c8d95..b8d2b867 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/areacello.py @@ -2,10 +2,11 @@ compute Grid-Cell Area for Ocean Variables areacello """ import xarray -import logging -from e3sm_to_cmip import mpas -from e3sm_to_cmip.util import print_message, setup_cmor +from e3sm_to_cmip import mpas, util +from e3sm_to_cmip.util import print_message +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPAS_mesh', 'MPAS_map'] @@ -41,7 +42,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -65,8 +66,7 @@ def handle(infiles, tables, user_input_path, **kwargs): # area_b is in square radians, so need to multiply by the earth_radius**2 ds[VAR_NAME] = earth_radius**2*area_b*ds[VAR_NAME] - setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, - user_input_path=user_input_path) + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'latitude', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py index 351efc6a..b6d097ba 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/fsitherm.py @@ -5,9 +5,11 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas + +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -43,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(f'Simple CMOR output not supported for {VAR_NAME}', 'error') return None - logging.info(f'Starting {VAR_NAME}') + logger.info(f'Starting {VAR_NAME}') mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -60,7 +62,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py index a7c0d281..6d4fdf25 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfds.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(f'Simple CMOR output not supported for {VAR_NAME}', 'error') return None - logging.info(f'Starting {VAR_NAME}') + logger.info(f'Starting {VAR_NAME}') mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -73,7 +74,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py index baf5c2e1..83cd202f 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/hfsifrazil.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(f'Simple CMOR output not supported for {VAR_NAME}', 'error') return None - logging.info(f'Starting {VAR_NAME}') + logger.info(f'Starting {VAR_NAME}') timeSeriesFiles = infiles['MPASO'] mappingFileName = infiles['MPAS_map'] @@ -76,7 +77,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py index 6030ccd1..4a7dc32c 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/masscello.py @@ -5,10 +5,11 @@ from __future__ import absolute_import, division, print_function import xarray -import logging import netCDF4 +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPASO_namelist', 'MPAS_mesh', 'MPAS_map'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles['MPASO_namelist'] meshFileName = infiles['MPAS_mesh'] @@ -77,7 +78,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds[VAR_NAME] = ds[VAR_NAME].where( ds[VAR_NAME] != netCDF4.default_fillvals['f4'], 0.) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py index 8155fcea..085fbfd5 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/masso.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPASO_namelist', 'MPAS_mesh'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles['MPASO_namelist'] meshFileName = infiles['MPAS_mesh'] @@ -69,7 +70,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.add_time(ds, dsIn) ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py index 43309b1e..079eeadb 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/mlotst.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -66,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.add_mask(ds, cellMask2D) ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py index 1e255108..70d28ec1 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/msftmz.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPASO_MOC_regions', 'MPASO_namelist'] @@ -47,7 +48,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -76,7 +77,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = ds.rename({'moc': VAR_NAME}) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) region = ['global_ocean', 'atlantic_arctic_ocean'] diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py index 792fa1b8..0c233744 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/pbo.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPASO_namelist', 'MPAS_mesh', 'MPAS_map', 'PSL'] @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles['MPASO_namelist'] meshFileName = infiles['MPAS_mesh'] @@ -84,7 +85,7 @@ def handle(infiles, tables, user_input_path, **kwargs): with xarray.open_mfdataset(pslFileNames, concat_dim='time') as dsIn: ds[VAR_NAME] = ds[VAR_NAME] + dsIn.PSL.values - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py index 304ac8c8..ed694d01 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/pso.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPASO_namelist', 'MPAS_mesh', 'MPAS_map', 'PSL'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) namelistFileName = infiles['MPASO_namelist'] meshFileName = infiles['MPAS_mesh'] @@ -80,7 +81,7 @@ def handle(infiles, tables, user_input_path, **kwargs): with xarray.open_mfdataset(pslFileNames, concat_dim='time') as dsIn: ds[VAR_NAME] = ds[VAR_NAME] + dsIn.PSL.values - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py index af9e28fa..f4e368e0 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sfdsi.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -61,7 +62,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py index f9a38943..6c7ad22d 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/siconc.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -63,7 +64,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py index ec514576..158d8df9 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/simass.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -41,7 +42,7 @@ def handle(infiles, tables, user_input_path, **kwargs): the name of the processed variable after processing is complete """ msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -62,7 +63,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py index 1542d421..e1d31269 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnmass.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py index 13de8a81..57ebadb4 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sisnthick.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py index 706ea8e1..7868a573 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitemptop.py @@ -5,11 +5,11 @@ from __future__ import absolute_import, division, print_function -import logging - import xarray +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -48,7 +48,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py index e9b66c89..70cf0ae5 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sithick.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -66,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py index 591ff362..4645555b 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sitimefrac.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -65,7 +66,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py index b3105fbf..a3bc3c1a 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/siu.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -69,7 +70,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py index 4fd8b4f4..0306ffa4 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/siv.py @@ -6,10 +6,11 @@ from __future__ import absolute_import, division, print_function import xarray -import logging - -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_console=True, propagate=False) + # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASSI', 'MPAS_mesh', 'MPAS_map'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -70,7 +71,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasseaice', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='seaice') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py index d622383e..5c5f3a20 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/so.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py index 81626534..ed411d77 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sob.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py index 7f3f0d5c..16dd324d 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/soga.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -70,7 +71,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.add_time(ds, dsIn) ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py index 0d4b731e..0367f837 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sos.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py index be153e90..932bcc9e 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/sosga.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -66,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds.compute() ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py index dea84253..1e78aa7c 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauuo.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -61,7 +62,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py index 032a2436..703d753d 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tauvo.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -61,7 +62,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py index f35af82c..2d2649c3 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetao.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -43,8 +44,8 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(msg) return - msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + msg = f'Starting {__name__}' + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py index 632c0a21..3826740a 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/thetaoga.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -70,7 +71,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.add_time(ds, dsIn) ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py index 01e6420e..a1500075 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/thkcello.py @@ -3,11 +3,12 @@ """ import xarray -import logging import netCDF4 +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas -from e3sm_to_cmip.util import print_message, setup_cmor +from e3sm_to_cmip import mpas, util +from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -42,19 +43,25 @@ def handle(infiles, tables, user_input_path, **kwargs): print_message(msg) return - msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + msg = f'Starting {__name__}' + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] + msg = f' Calling xarray.open_dataset()' + logger.info(msg) + dsMesh = xarray.open_dataset(meshFileName, mask_and_scale=False) _, cellMask3D = mpas.get_mpaso_cell_masks(dsMesh) variableList = ['timeMonthly_avg_layerThickness', 'xtime_startMonthly', 'xtime_endMonthly'] + msg = f' Calling mpas.open_mfdataset()' + logger.info(msg) + ds = xarray.Dataset() with mpas.open_mfdataset(timeSeriesFiles, variableList) as dsIn: ds[VAR_NAME] = \ @@ -71,8 +78,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds[VAR_NAME] = ds[VAR_NAME].where( ds[VAR_NAME] != netCDF4.default_fillvals['f4'], 0.) - setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, - user_input_path=user_input_path) + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py index 8c08df06..ac5ce720 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tob.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py index 4bc4d36d..a1e6b9ae 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tos.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py index 34bb9e96..7d390051 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/tosga.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -66,7 +67,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds.compute() ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py index da0562e4..b6a5bb1d 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/uo.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py index 0085a97a..0317ebc7 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/vo.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -44,7 +45,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py index 9a228bea..3e087dfb 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/volcello.py @@ -3,11 +3,12 @@ """ import xarray -import logging import netCDF4 +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas -from e3sm_to_cmip.util import print_message, setup_cmor +from e3sm_to_cmip import mpas, util +from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -43,7 +44,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -85,8 +86,7 @@ def handle(infiles, tables, user_input_path, **kwargs): # multiply variables in this order so they don't get transposed ds[VAR_NAME] = ds[VAR_NAME]*earth_radius**2*area_b - setup_cmor(var_name=VAR_NAME, table_path=tables, table_name=TABLE, - user_input_path=user_input_path) + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py index b2e838a7..01c406fa 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/volo.py @@ -6,9 +6,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] timeSeriesFiles = infiles['MPASO'] @@ -67,7 +68,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds.compute() - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py index 2a4ab15d..a7b87ffd 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/wfo.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_map'] @@ -46,7 +47,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) mappingFileName = infiles['MPAS_map'] timeSeriesFiles = infiles['MPASO'] @@ -71,7 +72,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py index 59995dca..1b2f98f2 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/wo.py @@ -5,10 +5,11 @@ from __future__ import absolute_import, division, print_function import xarray -import logging import numpy +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -70,7 +71,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py index 42fe931c..54dea42d 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/zhalfo.py @@ -5,10 +5,11 @@ from __future__ import absolute_import, division, print_function import xarray -import logging import numpy +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed RAW_VARIABLES = ['MPASO', 'MPAS_mesh', 'MPAS_map'] @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -76,7 +77,7 @@ def handle(infiles, tables, user_input_path, **kwargs): zLayerBot = (zLayerBot - layerThickness.isel(nVertLevels=zIndex)).where(mask) zLayerBot.compute() - # print('done zLayerBot {}/{}'.format(zIndex+1, nVertLevels)) + logger.info(f"done zLayerBot {zIndex+1}/{nVertLevels}") slices.append(zLayerBot) maskSlices.append(mask) ds[VAR_NAME] = xarray.concat(slices, dim='olevhalf') @@ -85,12 +86,13 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = ds.transpose('Time', 'olevhalf', 'nCells') ds = mpas.add_time(ds, dsIn) ds.compute() + logger.info(f"returned from ds.compute()") ds = mpas.remap(ds, 'mpasocean', mappingFileName) depth_coord_half = numpy.zeros(nVertLevels+1) depth_coord_half[1:] = dsMesh.refBottomDepth.values - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py b/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py index 7903d818..b2686bbc 100644 --- a/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py +++ b/e3sm_to_cmip/cmor_handlers/mpas_vars/zos.py @@ -5,9 +5,10 @@ from __future__ import absolute_import, division, print_function import xarray -import logging +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) -from e3sm_to_cmip import mpas +from e3sm_to_cmip import mpas, util from e3sm_to_cmip.util import print_message # 'MPAS' as a placeholder for raw variables needed @@ -45,7 +46,7 @@ def handle(infiles, tables, user_input_path, **kwargs): return msg = 'Starting {name}'.format(name=__name__) - logging.info(msg) + logger.info(msg) meshFileName = infiles['MPAS_mesh'] mappingFileName = infiles['MPAS_map'] @@ -69,7 +70,7 @@ def handle(infiles, tables, user_input_path, **kwargs): ds = mpas.remap(ds, 'mpasocean', mappingFileName) - mpas.setup_cmor(VAR_NAME, tables, user_input_path, component='ocean') + util.setup_cmor(VAR_NAME, tables, TABLE, user_input_path) # create axes axes = [{'table_entry': 'time', diff --git a/e3sm_to_cmip/cmor_handlers/utils.py b/e3sm_to_cmip/cmor_handlers/utils.py index 8f946e8e..f67232a7 100644 --- a/e3sm_to_cmip/cmor_handlers/utils.py +++ b/e3sm_to_cmip/cmor_handlers/utils.py @@ -11,11 +11,16 @@ LEGACY_HANDLER_DIR_PATH, MPAS_HANDLER_DIR_PATH, ) -from e3sm_to_cmip._logger import _setup_logger from e3sm_to_cmip.cmor_handlers.handler import VarHandler from e3sm_to_cmip.util import _get_table_for_non_monthly_freq -logger = _setup_logger(__name__) +from e3sm_to_cmip._logger import e2c_logger + +def instantiate_h_utils_logger(): + global logger + + logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) + # Type aliases Frequency = Literal["mon", "day", "6hrLev", "6hrPlev", "6hrPlevPt", "3hr", "1hr"] @@ -55,6 +60,7 @@ def load_all_handlers( KeyError If no handlers are defined for a CMIP6 variable in `handlers.yaml`. """ + handlers_by_var: Dict[str, List[Dict[str, Any]]] = _get_handlers_by_var() missing_handlers: List[str] = [] @@ -172,6 +178,7 @@ def derive_handlers( If a handler could not be derived for a CMIP6 variable using the existing E3SM variables. """ + # TODO: Refactor the function parameters. handlers_by_var: Dict[str, List[Dict[str, Any]]] = _get_handlers_by_var() derived_handlers: List[Dict[str, Any]] = [] @@ -360,15 +367,15 @@ def _get_handlers_from_modules(path: str) -> Dict[str, List[Dict[str, Any]]]: "handler.py", "utils.py", ]: - var = file.split(".")[0] - filepath = os.path.join(root, file) - module = _get_handler_module(var, filepath) + module_name = file.split(".")[0] + module_path = os.path.join(root, file) + module = SourceFileLoader(module_name, module_path).load_module() # NOTE: The value is set to a list with a single dict entry # so that it is compatible with the data structure for storing # all var handlers (which can have multiple handlers per # variable). - handlers[var] = [ + handlers[module_name] = [ { "name": module.VAR_NAME, "units": module.VAR_UNITS, @@ -385,22 +392,3 @@ def _get_handlers_from_modules(path: str) -> Dict[str, List[Dict[str, Any]]]: return handlers -def _get_handler_module(module_name: str, module_path: str): - """Get the variable handler Python module. - - Parameters - ---------- - module_name : str - The name of the module, which should be the key of the variable (e.g., - "orog"). - module_path : str - The absolute path to the variable handler Python module. - - Returns - ------- - module - The module. - """ - module = SourceFileLoader(module_name, module_path).load_module() - - return module diff --git a/e3sm_to_cmip/cmor_handlers/vars/areacella.py b/e3sm_to_cmip/cmor_handlers/vars/areacella.py index dcd4d0e7..f23d8b8b 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/areacella.py +++ b/e3sm_to_cmip/cmor_handlers/vars/areacella.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import logging import os import numpy as np @@ -12,11 +11,11 @@ import cmor from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger from e3sm_to_cmip.mpas import write_netcdf -from e3sm_to_cmip.util import print_message +from e3sm_to_cmip.util import setup_cmor, print_message -logger = _setup_logger(__name__) +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) # list of raw variable names needed RAW_VARIABLES = [str("area")] @@ -56,46 +55,30 @@ def handle_simple(infiles): def handle(infiles, tables, user_input_path, table, logdir): - msg = "{}: Starting".format(VAR_NAME) - logger.info(msg) + logger.info(f"{VAR_NAME}: Starting") # check that we have some input files for every variable zerofiles = False for variable in RAW_VARIABLES: if len(infiles[variable]) == 0: - msg = "{}: Unable to find input files for {}".format(VAR_NAME, variable) + msg = f"{VAR_NAME}: Unable to find input files for {variable}" print_message(msg) - logging.error(msg) + logger.error(msg) zerofiles = True if zerofiles: return None - # Create the logging directory and setup cmor - if logdir: - logpath = logdir - else: - outpath, _ = os.path.split(logger.__dict__["handlers"][0].baseFilename) - logpath = os.path.join(outpath, "cmor_logs") - os.makedirs(logpath, exist_ok=True) + setup_cmor(VAR_NAME, tables, TABLE, user_input_path) - logfile = os.path.join(logpath, VAR_NAME + ".log") - - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - - cmor.dataset_json(str(user_input_path)) - cmor.load_table(str(TABLE)) - - msg = "{}: CMOR setup complete".format(VAR_NAME) - logging.info(msg) + logger.info(f"{VAR_NAME}: CMOR setup complete") # extract data from the input file - msg = "areacella: loading area" - logger.info(msg) + logger.info("areacella: loading area") filename = infiles["area"][0] if not os.path.exists(filename): - raise IOError("File not found: {}".format(filename)) + raise IOError(f"File not found: {filename}") ds = xr.open_dataset(filename, decode_times=False) @@ -108,8 +91,7 @@ def handle(infiles, tables, user_input_path, table, logdir): "area": ds["area"], } - msg = "{name}: loading axes".format(name=VAR_NAME) - logger.info(msg) + logger.info(f"{VAR_NAME}: loading axes") axes = [ { @@ -126,8 +108,7 @@ def handle(infiles, tables, user_input_path, table, logdir): }, ] - msg = "areacella: running CMOR" - logging.info(msg) + logger.info(f"{VAR_NAME}: running CMOR") axis_ids = list() for axis in axes: @@ -139,12 +120,10 @@ def handle(infiles, tables, user_input_path, table, logdir): outdata = data["area"].values * pow(RADIUS, 2) cmor.write(varid, outdata) - msg = "{}: write complete, closing".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: write complete, closing") cmor.close() - msg = "{}: file close complete".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: file close complete") - return "areacella" + return VAR_NAME diff --git a/e3sm_to_cmip/cmor_handlers/vars/clisccp.py b/e3sm_to_cmip/cmor_handlers/vars/clisccp.py index 05368eb8..f07b4ddb 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/clisccp.py +++ b/e3sm_to_cmip/cmor_handlers/vars/clisccp.py @@ -5,7 +5,6 @@ """ from __future__ import absolute_import, annotations, division, unicode_literals -import logging import os from typing import Dict, List, Union @@ -13,10 +12,10 @@ import xarray as xr import cmor -from e3sm_to_cmip._logger import _setup_logger -from e3sm_to_cmip.util import print_message +from e3sm_to_cmip.util import setup_cmor, print_message -logger = _setup_logger(__name__) +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) # list of raw variable names needed RAW_VARIABLES = [str("FISCCP1_COSP")] @@ -59,35 +58,23 @@ def handle( # noqa: C901 If CMORizing was successful, return the output CMIP variable name to indicate success. If failed, return None .""" - logging.info(f"Starting {VAR_NAME}") + logger.info(f"Starting {VAR_NAME}") nonzero = False for variable in RAW_VARIABLES: if len(vars_to_filepaths[variable]) == 0: msg = f"{variable}: Unable to find input files for {RAW_VARIABLES}" print_message(msg) - logging.error(msg) + logger.error(msg) nonzero = True if nonzero: return None - msg = f"{VAR_NAME}: running with input files: {vars_to_filepaths}" - logger.debug(msg) + logger.debug(f"{VAR_NAME}: running with input files: {vars_to_filepaths}") - if logdir is not None: - logfile = logfile = os.path.join(logdir, VAR_NAME + ".log") - else: - logfile = os.path.join(os.getcwd(), "logs") - if not os.path.exists(logfile): - os.makedirs(logfile) - logfile = os.path.join(logfile, VAR_NAME + ".log") + setup_cmor(VAR_NAME, tables, TABLE, metadata_path) - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - cmor.dataset_json(metadata_path) - cmor.load_table(TABLE) - - msg = f"{VAR_NAME}: CMOR setup complete" - logger.info(msg) + logger.info(f"{VAR_NAME}: CMOR setup complete") data: Dict[str, Union[np.ndarray, xr.DataArray]] = {} @@ -164,8 +151,7 @@ def handle( # noqa: C901 varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids) # write out the data - msg = f"{VAR_NAME}: time {data['time_bnds'][0][0]:1.1f} - {data['time_bnds'][-1][-1]:1.1f}" - logger.info(msg) + logger.info(f"{VAR_NAME}: time {data['time_bnds'][0][0]:1.1f} - {data['time_bnds'][-1][-1]:1.1f}") cmor.write( varid, @@ -174,11 +160,9 @@ def handle( # noqa: C901 time_bnds=data["time_bnds"], ) - msg = f"{VAR_NAME}: write complete, closing" - logger.info(msg) + logger.info(f"{VAR_NAME}: write complete, closing") cmor.close() - msg = f"{VAR_NAME}: file close complete" - logger.info(msg) + logger.info(f"{VAR_NAME}: file close complete") return VAR_NAME diff --git a/e3sm_to_cmip/cmor_handlers/vars/orog.py b/e3sm_to_cmip/cmor_handlers/vars/orog.py index 6767e71b..c17aa02e 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/orog.py +++ b/e3sm_to_cmip/cmor_handlers/vars/orog.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import logging import os import numpy as np @@ -12,11 +11,11 @@ import cmor from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger from e3sm_to_cmip.mpas import write_netcdf -from e3sm_to_cmip.util import print_message +from e3sm_to_cmip.util import setup_cmor, print_message -logger = _setup_logger(__name__) +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) # list of raw variable names needed RAW_VARIABLES = [str("PHIS")] @@ -56,8 +55,7 @@ def handle_simple(infiles): def handle(infiles, tables, user_input_path, table, logdir): - msg = f"{VAR_NAME}: Starting" - logger.info(msg) + logger.info(f"{VAR_NAME}: Starting") # check that we have some input files for every variable zerofiles = False @@ -65,32 +63,17 @@ def handle(infiles, tables, user_input_path, table, logdir): if len(infiles[variable]) == 0: msg = f"{VAR_NAME}: Unable to find input files for {variable}" print_message(msg) - logging.error(msg) + logger.error(msg) zerofiles = True if zerofiles: return None - # Create the logging directory and setup cmor - if logdir: - logpath = logdir - else: - outpath, _ = os.path.split(logger.__dict__["handlers"][0].baseFilename) - logpath = os.path.join(outpath, "cmor_logs") - os.makedirs(logpath, exist_ok=True) + setup_cmor(VAR_NAME, tables, TABLE, user_input_path) - logfile = os.path.join(logpath, VAR_NAME + ".log") - - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - - cmor.dataset_json(str(user_input_path)) - cmor.load_table(str(TABLE)) - - msg = "{}: CMOR setup complete".format(VAR_NAME) - logging.info(msg) + logger.info(f"{VAR_NAME}: CMOR setup complete") # extract data from the input file - msg = "orog: loading PHIS" - logger.info(msg) + logger.info("orog: loading PHIS") filename = infiles["PHIS"][0] @@ -108,8 +91,7 @@ def handle(infiles, tables, user_input_path, table, logdir): "PHIS": ds["PHIS"], } - msg = f"{VAR_NAME}: loading axes" - logger.info(msg) + logger.info(f"{VAR_NAME}: loading axes") axes = [ { @@ -126,8 +108,7 @@ def handle(infiles, tables, user_input_path, table, logdir): }, ] - msg = "orog: running CMOR" - logging.info(msg) + logger.info("orog: running CMOR") axis_ids = list() for axis in axes: @@ -139,12 +120,10 @@ def handle(infiles, tables, user_input_path, table, logdir): outdata = data["PHIS"].values / GRAV cmor.write(varid, outdata) - msg = "{}: write complete, closing".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: write complete, closing") cmor.close() - msg = "{}: file close complete".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: file close complete") - return "orog" + return VARNAME diff --git a/e3sm_to_cmip/cmor_handlers/vars/sftlf.py b/e3sm_to_cmip/cmor_handlers/vars/sftlf.py index 0a5f4da2..0401458e 100644 --- a/e3sm_to_cmip/cmor_handlers/vars/sftlf.py +++ b/e3sm_to_cmip/cmor_handlers/vars/sftlf.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals import json -import logging import os import numpy as np @@ -12,11 +11,11 @@ import cmor from e3sm_to_cmip import resources -from e3sm_to_cmip._logger import _setup_logger from e3sm_to_cmip.mpas import write_netcdf -from e3sm_to_cmip.util import print_message +from e3sm_to_cmip.util import setup_cmor, print_message -logger = _setup_logger(__name__) +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) # list of raw variable names needed RAW_VARIABLES = [str("LANDFRAC")] @@ -55,8 +54,7 @@ def handle_simple(infiles): def handle(infiles, tables, user_input_path, table, logdir): - msg = f"{VAR_NAME}: Starting" - logger.info(msg) + logger.info(f"{VAR_NAME}: Starting") # check that we have some input files for every variable zerofiles = False @@ -64,32 +62,17 @@ def handle(infiles, tables, user_input_path, table, logdir): if len(infiles[variable]) == 0: msg = f"{VAR_NAME}: Unable to find input files for {variable}" print_message(msg) - logging.error(msg) + logger.error(msg) zerofiles = True if zerofiles: return None - # Create the logging directory and setup cmor - if logdir: - logpath = logdir - else: - outpath, _ = os.path.split(logger.__dict__["handlers"][0].baseFilename) - logpath = os.path.join(outpath, "cmor_logs") - os.makedirs(logpath, exist_ok=True) + setup_cmor(VAR_NAME, tables, TABLE, user_input_path) - logfile = os.path.join(logpath, VAR_NAME + ".log") - - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - - cmor.dataset_json(str(user_input_path)) - cmor.load_table(str(TABLE)) - - msg = "{}: CMOR setup complete".format(VAR_NAME) - logging.info(msg) + logger.info(f"{VAR_NAME}: CMOR setup complete") # extract data from the input file - msg = "sftlf: loading LANDFRAC" - logger.info(msg) + logger.info("sftlf: loading LANDFRAC") filename = infiles["LANDFRAC"][0] @@ -107,8 +90,7 @@ def handle(infiles, tables, user_input_path, table, logdir): "LANDFRAC": ds["LANDFRAC"], } - msg = f"{VAR_NAME}: loading axes" - logger.info(msg) + logger.info(f"{VAR_NAME}: loading axes") axes = [ { @@ -125,8 +107,7 @@ def handle(infiles, tables, user_input_path, table, logdir): }, ] - msg = "sftlf: running CMOR" - logging.info(msg) + logger.info(f"{VAR_NAME}: running CMOR") axis_ids = list() for axis in axes: @@ -138,12 +119,10 @@ def handle(infiles, tables, user_input_path, table, logdir): outdata = data["LANDFRAC"].values * 100.0 cmor.write(varid, outdata) - msg = "{}: write complete, closing".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: write complete, closing") cmor.close() - msg = "{}: file close complete".format(VAR_NAME) - logger.debug(msg) + logger.debug(f"{VAR_NAME}: file close complete") - return "sftlf" + return VAR_NAME diff --git a/e3sm_to_cmip/mpas.py b/e3sm_to_cmip/mpas.py index 755db9e1..0e56c98e 100644 --- a/e3sm_to_cmip/mpas.py +++ b/e3sm_to_cmip/mpas.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function import argparse -import logging import multiprocessing import os import re @@ -23,21 +22,22 @@ import xarray from dask.diagnostics import ProgressBar +from e3sm_to_cmip._logger import e2c_logger +logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) def run_ncremap_cmd(args, env): - logtext = f"mpas.py: remap: ncremap args = {args}" - logging.info(logtext) proc = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) (out, err) = proc.communicate() - logging.info(out) + if proc.returncode: - print("Error running ncremap command: {}".format(" ".join(args))) + arglist = " ".join(args) + logger.error(f"Error running ncremap command: {arglist}") print(err.decode("utf-8")) raise subprocess.CalledProcessError( # type: ignore - "ncremap returned {}".format(proc.returncode) # type: ignore + f"ncremap returned {proc.returncode}" # type: ignore ) @@ -46,6 +46,9 @@ def remap_seaice_sgs(inFileName, outFileName, mappingFileName, renorm_threshold= ds_in = xarray.open_dataset(inFileName, decode_times=False) outFilePath = f"{outFileName}sub" os.makedirs(outFilePath) + + logger.info(f"Calling run_ncremap_cmd for each ds_slice in {range(ds_in.sizes['time'])}") + for t_index in range(ds_in.sizes["time"]): ds_slice = ds_in.isel(time=slice(t_index, t_index + 1)) ds_slice.to_netcdf(f"{outFilePath}/temp_in{t_index}.nc") @@ -77,13 +80,17 @@ def remap_seaice_sgs(inFileName, outFileName, mappingFileName, renorm_threshold= def remap(ds, pcode, mappingFileName, threshold=0.0): """Use ncreamp to remap the xarray Dataset to a new target grid""" + delete_tempfiles = True + # write the dataset to a temp file inFileName = _get_temp_path() outFileName = _get_temp_path() if "depth" in ds.dims: + logger.info(f"Calling ds.transpose") ds = ds.transpose("time", "depth", "nCells", "nbnd") + logger.info(f"Calling write_netcdf() (inFileName={inFileName})") write_netcdf(ds, inFileName, unlimited="time") # set an environment variable to make sure we're not using czender's @@ -111,6 +118,7 @@ def remap(ds, pcode, mappingFileName, threshold=0.0): inFileName, outFileName, ] + logger.info(f"Calling run_ncremap_cmd with args {args}") run_ncremap_cmd(args, env) elif pcode == "mpasseaice": # MPAS-Seaice is a special case because the of the time-varying SGS field @@ -127,9 +135,10 @@ def remap(ds, pcode, mappingFileName, threshold=0.0): ds.load() - # remove the temporary files - os.remove(inFileName) - os.remove(outFileName) + if delete_tempfiles: + # remove the temporary files + os.remove(inFileName) + os.remove(outFileName) return ds @@ -321,7 +330,6 @@ def open_mfdataset( concat_dim="Time", mask_and_scale=False, chunks=chunks, - lock=False, ) if variableList is not None: @@ -351,6 +359,9 @@ def write_netcdf(ds, fileName, fillValues=netCDF4.default_fillvals, unlimited=No encodingDict = {} variableNames = list(ds.data_vars.keys()) + list(ds.coords.keys()) for variableName in variableNames: + if "_FillValue" in ds[variableName].attrs: + # There's already a fill value attribute so don't add a new one + continue isNumeric = np.issubdtype(ds[variableName].dtype, np.number) if isNumeric: dtype = ds[variableName].dtype @@ -395,29 +406,9 @@ def convert_namelist_to_dict(fileName): return nml -def setup_cmor(varname, tables, user_input_path, component="ocean", table=None): - """Set up CMOR for MPAS-Ocean or MPAS-Seaice""" - logfile = os.path.join(os.getcwd(), "cmor_logs") - if not os.path.exists(logfile): - os.makedirs(logfile) - logfile = os.path.join(logfile, varname + ".log") - cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - cmor.dataset_json(str(user_input_path)) - if table is None: - if component == "ocean": - table = "CMIP6_Omon.json" - elif component == "seaice": - table = "CMIP6_SImon.json" - else: - raise ValueError("Unexpected component {}".format(component)) - try: - cmor.load_table(table) - except Exception: - raise ValueError("Unable to load table from {}".format(varname)) - - def write_cmor(axes, ds, varname, varunits, d2f=True, **kwargs): """Write a time series of a variable in the format expected by CMOR""" + axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) @@ -437,6 +428,10 @@ def write_cmor(axes, ds, varname, varunits, d2f=True, **kwargs): str(varname), str(varunits), axis_ids, missing_value=fillValue, **kwargs ) + # adding "shuffle" reduced size on disk by 14%, need to test for performance impact. + # if varname not in [ "lat", "lev", "lon" ]: + # cmor.set_deflate(varid, True, True, 1) + # write out the data try: if "time" not in ds.dims: @@ -449,7 +444,7 @@ def write_cmor(axes, ds, varname, varunits, d2f=True, **kwargs): time_bnds=ds.time_bnds.values, ) except Exception as error: - logging.exception(f"Error in cmor.write for {varname}") + logger.exception(f"Error in cmor.write for {varname}") raise Exception(error) finally: cmor.close(varid) diff --git a/e3sm_to_cmip/util.py b/e3sm_to_cmip/util.py index 8c57c995..723b76ca 100644 --- a/e3sm_to_cmip/util.py +++ b/e3sm_to_cmip/util.py @@ -14,10 +14,12 @@ import yaml from tqdm import tqdm -from e3sm_to_cmip._logger import _setup_logger +from e3sm_to_cmip._logger import e2c_logger -logger = _setup_logger(__name__) +def instantiate_util_logger(): + global logger + logger = e2c_logger(name=__name__, set_log_level="INFO", to_logfile=True, propagate=False) ATMOS_TABLES = [ "CMIP6_Amon.json", @@ -103,24 +105,25 @@ def print_message(message, status="error"): # ------------------------------------------------------------------ -def setup_cmor(var_name, table_path, table_name, user_input_path): +def setup_cmor(varname, table_path, table_name, user_input_path): """ Sets up cmor and logging for a single handler """ - var_name = str(var_name) - table_path = str(table_path) - table_name = str(table_name) - user_input_path = str(user_input_path) - logfile = os.path.join(os.getcwd(), "logs") + logfile = os.path.join(os.getcwd(), "cmor_logs") if not os.path.exists(logfile): os.makedirs(logfile) + logfile = os.path.join(logfile, varname + ".log") - logfile = os.path.join(logfile, var_name + ".log") cmor.setup(inpath=table_path, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) - cmor.dataset_json(user_input_path) - cmor.load_table(table_name) + cmor.dataset_json(str(user_input_path)) + + try: + cmor.load_table(table_name) + except Exception: + raise ValueError(f"Unable to load table {table_name} for {varname}") + # ------------------------------------------------------------------ @@ -561,7 +564,7 @@ def find_mpas_files(component, path, map_path=None): # noqa: C901 pattern_v2 = "mocBasinsAndTransects" for infile in contents: if pattern_v1 in infile or pattern_v2 in infile: - logger.info(f"component mpas0_moc_regions found: {infile}") + logger.info(f"component mpaso_moc_regions found: {infile}") return os.path.abspath(os.path.join(path, infile)) raise IOError("Unable to find mpaso_moc_regions in the input directory") From 580a1a4f65f325f428edc89360cef6976538c5df Mon Sep 17 00:00:00 2001 From: Tony Bartoletti Date: Thu, 27 Jun 2024 15:03:35 -0700 Subject: [PATCH 2/9] revert handler cmor setup to logfile --- e3sm_to_cmip/cmor_handlers/handler.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/e3sm_to_cmip/cmor_handlers/handler.py b/e3sm_to_cmip/cmor_handlers/handler.py index 655df5c1..4212745f 100644 --- a/e3sm_to_cmip/cmor_handlers/handler.py +++ b/e3sm_to_cmip/cmor_handlers/handler.py @@ -322,18 +322,18 @@ def _setup_cmor_module( logdir : str | None The optional log directory. """ - # if log_dir is not None: - # logpath = log_dir - # else: - # cwd = os.getcwd() - # logpath = os.path.join(cwd, "cmor_logs") + if log_dir is not None: + logpath = log_dir + else: + cwd = os.getcwd() + logpath = os.path.join(cwd, "cmor_logs") - # os.makedirs(logpath, exist_ok=True) - # logfile = os.path.join(logpath, var_name + ".log") + os.makedirs(logpath, exist_ok=True) + logfile = os.path.join(logpath, var_name + ".log") print("DEBUG: EMPLOYED HANDLER.PY _setup_cmor_module()", flush=True) - cmor.setup(inpath=tables_path, netcdf_file_action=cmor.CMOR_REPLACE,logfile="/dev/null") + cmor.setup(inpath=tables_path, netcdf_file_action=cmor.CMOR_REPLACE,logfile=logfile) cmor.dataset_json(metadata_path) cmor.load_table(self.table) From 0a380d4d64c54bff57052f3c107edc75f0d8f51d Mon Sep 17 00:00:00 2001 From: Tony Bartoletti Date: Thu, 27 Jun 2024 15:27:29 -0700 Subject: [PATCH 3/9] add commentary on do_shuffle and delete_tempfiles as future flags --- e3sm_to_cmip/mpas.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e3sm_to_cmip/mpas.py b/e3sm_to_cmip/mpas.py index 0e56c98e..615197ce 100644 --- a/e3sm_to_cmip/mpas.py +++ b/e3sm_to_cmip/mpas.py @@ -80,6 +80,7 @@ def remap_seaice_sgs(inFileName, outFileName, mappingFileName, renorm_threshold= def remap(ds, pcode, mappingFileName, threshold=0.0): """Use ncreamp to remap the xarray Dataset to a new target grid""" + # TODO: Make this a global commandline switch delete_tempfiles = True # write the dataset to a temp file @@ -429,7 +430,8 @@ def write_cmor(axes, ds, varname, varunits, d2f=True, **kwargs): ) # adding "shuffle" reduced size on disk by 14%, need to test for performance impact. - # if varname not in [ "lat", "lev", "lon" ]: + # TODO: Pass a commandline arg "--shuffle" down to this point. + # if do_shuffle and varname not in [ "lat", "lev", "lon" ]: # cmor.set_deflate(varid, True, True, 1) # write out the data From 20feb790b9f7eee1aff32ad78d9d8c71f607a626 Mon Sep 17 00:00:00 2001 From: Tony Bartoletti Date: Thu, 27 Jun 2024 15:49:46 -0700 Subject: [PATCH 4/9] minor cleanup --- e3sm_to_cmip/__main__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/e3sm_to_cmip/__main__.py b/e3sm_to_cmip/__main__.py index 973de7b0..314a2bae 100755 --- a/e3sm_to_cmip/__main__.py +++ b/e3sm_to_cmip/__main__.py @@ -311,7 +311,8 @@ def _parse_args(self, args: Optional[List[str]]) -> CLIArguments: sys.exit(1) # Parse the arguments and perform validation. - parsed_args = argparser.parse_args(args_to_parse) # (exits here if args == "-h" or "--help" or "--version") + # NOTE: exits here if args == "-h" or "--help" or "--version", else validate + parsed_args = argparser.parse_args(args_to_parse) self._validate_parsed_args(parsed_args) @@ -785,7 +786,9 @@ def _run_serial(self) -> int: # noqa: C901 1 if an error occurs, else 0 """ + # TODO: Make this a command-line flag. do_pbar = False + try: num_handlers = len(self.handlers) num_success = 0 @@ -884,6 +887,7 @@ def _run_parallel(self) -> int: # noqa: C901 pool_res = list() will_run = [] + # NOTE: Make this a command-line flag. do_pbar = False for idx, handler in enumerate(self.handlers): @@ -992,8 +996,8 @@ def main(args: Optional[List[str]] = None): app = E3SMtoCMIP(args) - # These calls allow loggers that create default logfiles to avoid being - # instantiated by arguments "--help" or "--version". + # These calls allow module loggers that create default logfiles to avoid being + # instantiated by arguments "--help" or "--version" upon import. instantiate_util_logger() instantiate_h_utils_logger() instantiate_handler_logger() From 1aea912a918ecc5df16612de58c1de7871ff3852 Mon Sep 17 00:00:00 2001 From: Tony Bartoletti Date: Thu, 27 Jun 2024 15:56:35 -0700 Subject: [PATCH 5/9] minor cleanup --- e3sm_to_cmip/cmor_handlers/handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/e3sm_to_cmip/cmor_handlers/handler.py b/e3sm_to_cmip/cmor_handlers/handler.py index 4212745f..87ade4e0 100644 --- a/e3sm_to_cmip/cmor_handlers/handler.py +++ b/e3sm_to_cmip/cmor_handlers/handler.py @@ -331,9 +331,7 @@ def _setup_cmor_module( os.makedirs(logpath, exist_ok=True) logfile = os.path.join(logpath, var_name + ".log") - print("DEBUG: EMPLOYED HANDLER.PY _setup_cmor_module()", flush=True) - - cmor.setup(inpath=tables_path, netcdf_file_action=cmor.CMOR_REPLACE,logfile=logfile) + cmor.setup(inpath=tables_path, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(metadata_path) cmor.load_table(self.table) From 25f6f127bec21f2a1db7740e54bb687d7f183988 Mon Sep 17 00:00:00 2001 From: Tony Bartoletti Date: Wed, 10 Jul 2024 10:21:08 -0700 Subject: [PATCH 6/9] modified e2e script to allow mpaso processing to cycle over YPF segments --- scripts/example_end_to_end_script.sh | 169 ++++++++++++--------------- 1 file changed, 74 insertions(+), 95 deletions(-) diff --git a/scripts/example_end_to_end_script.sh b/scripts/example_end_to_end_script.sh index 562a0b6c..8200dbfc 100644 --- a/scripts/example_end_to_end_script.sh +++ b/scripts/example_end_to_end_script.sh @@ -37,100 +37,6 @@ map_file=${e2c_path}/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc tables_path=${e2c_path}/cmor/cmip6-cmor-tables/Tables/ metadata_path=${e2c_path}/user_metadata.json -# NOTE: Space is not accepted in nco var list -## ------------------------------------------------------ -## TEST CASE - atm monthly h0 -## ------------------------------------------------------ -input_path=${model_data}/v2.eam_input -flags='-7 --dfl_lvl=1 --no_cll_msr' -raw_var_list="ICEFRAC,OCNFRAC,LANDFRAC,PHIS,hyam,hybm,hyai,hybi,TREFHT,TS,PSL,PS,U10,QREFHT,PRECC,PRECL,PRECSC,PRECSL,QFLX,TAUX,TAUY,LHFLX,CLDTOT,FLDS,FLNS,FSDS,FSNS,SHFLX,CLOUD,CLDICE,TGCLDIWP,CLDLIQ,TGCLDCWP,TMQ,FLNSC,FSNTOA,FSNT,FLNT,FLUTC,FSDSC,SOLIN,FSNSC,FSUTOA,FSUTOAC,AODABS,AODVIS,AREL,TREFMNAV,TREFMXAV,FISCCP1_COSP,CLDTOT_ISCCP,MEANCLDALB_ISCCP,MEANPTOP_ISCCP,CLD_CAL,CLDTOT_CAL,CLDLOW_CAL,CLDMED_CAL,CLDHGH_CAL" - -# 1. atm 2D variables -#-------------------------- -cmip_var_list="pfull, phalf, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, cl, clw, cli, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin, clisccp, cltisccp, albisccp, pctisccp, clcalipso, cltcalipso, cllcalipso, clmcalipso, clhcalipso" -ncclimo -P eam -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} -# CMORIZE Atmosphere monthly variables: 2D and model level 3D variables (CLOUD,CLDICE,CLDLIQ) -e3sm_to_cmip -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t $tables_path -u ${metadata_path} - -# 2. atm fixed variables -#-------------------------- -raw_var_list="area,PHIS,LANDFRAC" -cmip_var_list="areacella, sftlf, orog" -ncremap --map=${map_file} -v area,PHIS,LANDFRAC -I ${input_path} -O ${rgr_dir}/fixed_vars - -# CMORIZE Atmosphere fx variables -e3sm_to_cmip --realm fx -i ${rgr_dir}/fixed_vars -o $result_dir -v areacella, sftlf, orog -t ${tables_path} -u ${metadata_path} - -## ------------------------------------------------------ -## TEST CASE - atm 3D variables -## ------------------------------------------------------ -flags='-7 --dfl_lvl=1 --no_cll_msr' -raw_var_list="Q,O3,T,U,V,Z3,RELHUM,OMEGA" -cmip_var_list="hur, hus, ta, ua, va, wap, zg, o3" -ncclimo -P eam -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir_vert} -v ${raw_var_list} -i ${input_path} ${flags} -for file in `ls ${rgr_dir_vert}` -do - ncks --rgr xtr_mth=mss_val --vrt_fl=${e2c_path}/grids/vrt_remap_plev19.nc ${rgr_dir_vert}/$file ${rgr_dir}/$file -done - -# Note --start=$start --end=$end would not work with ncremap -#ncremap -P eam -j 1 --xtr_mth=mss_val --vrt_fl=${e2c_path}/grids/vrt_remap_plev19.nc -O ${rgr_dir} -v ${raw_var_list} -I ${rgr_dir_vert} ${flags} - -# CMORIZE Atmosphere monthly plev variables -e3sm_to_cmip -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} -u ${metadata_path} - -## ------------------------------------------------------ -## TEST CASE - atm high freq daily h1+ -## ------------------------------------------------------ -input_path=${model_data}/v2.eam.h1_input -flags='-7 --dfl_lvl=1 --no_cll_msr --clm_md=hfs' -raw_var_list="TREFHTMN,TREFHTMX,PRECT,TREFHT,FLUT,QREFHT" -cmip_var_list="tasmin, tasmax, tas, huss, rlut, pr" -rgr_dir=${result_dir}/rgr_day -native_dir=${result_dir}/native_day -ncclimo -P eam -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} - -# CMORIZE Atmosphere daily variables -e3sm_to_cmip --freq day -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} -u ${metadata_path} - -## ------------------------------------------------------ -## TEST CASE - atm high freq 3hrly h1+ -## ------------------------------------------------------ -input_path=${model_data}/v2.eam.h4_input -flags='-7 --dfl_lvl=1 --no_cll_msr --clm_md=hfs' -raw_var_list="PRECT" -rgr_dir=${result_dir}/rgr_3hr -native_dir=${result_dir}/native_3hr -cmip_var_list="pr" -ncclimo -P eam -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} - -# CMORIZE Atmosphere 3hrly variables -e3sm_to_cmip --freq 3hr -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} -u ${metadata_path} - -## ------------------------------------------------------ -## TEST CASE - land monthly h0 -## ------------------------------------------------------ -input_path=${model_data}/v2.elm_input/ -flags='-7 --dfl_lvl=1 --no_cll_msr' -raw_var_list="LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO" -cmip_var_list="mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai" -rgr_dir=${result_dir}/rgr_lnd -native_dir=${result_dir}/native_lnd - -# Note either include the extra variable landfrac or specify the file that has landfrac for subgrid scale mode to work. -ncclimo -P elm -j 1 --var_xtr=landfrac --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} -# Alternative ncclimo invocation -#ncclimo -P elm -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} --sgs_frc=${input_path}/v2.LR.historical_0101.elm.h0.1850-01.nc/landfrac - -#raw_var_list_elm_bgc="TOTLITC,CWDC,TOTPRODC,SOIL1C,SOIL2C,SOIL3C,^SOIL4C$,COL_FIRE_CLOSS,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" -# CMORIZE Land Monthly variables -e3sm_to_cmip -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} -u ${metadata_path} - -# CMORIZE Sea-ice Monthly variables -# Note the input folder for mpas sea ice files requires: -# 1. Monthly mean history files: e.g.,v2.LR.historical_0101.mpassi.hist.am.timeSeriesStatsMonthly.1850-01-01.nc -# 2. A restart file for meshes: can use the mpaso restart e.g., v2.LR.historical_0101.mpaso.rst.1855-01-01_00000.nc -e3sm_to_cmip -s --realm SImon --var-list siconc, sitemptop, sisnmass, sitimefrac, siu, siv, sithick, sisnthick, simass --map ${e2c_path}/maps/map_EC30to60E2r2_to_cmip6_180x360_aave.20220301.nc --input-path ${model_data}/v2.mpassi_input/ --output-path ${result_dir} --user-metadata ${metadata_path} --tables-path ${e2c_path}/cmor/cmip6-cmor-tables/Tables # CMORIZE Ocean Monthly variables # Note the input folder for mpas ocean files requires: @@ -140,6 +46,79 @@ e3sm_to_cmip -s --realm SImon --var-list siconc, sitemptop, sisnmass, sitimefrac # hfsifrazil requires 'config_density0' and 'config_frazil_heat_of_fusion' # 3. A restart file for mesh: e.g., v2.LR.historical_0101.mpaso.rst.1855-01-01_00000.nc # 4. A region masks file for MOC regions: EC30to60E2r2_mocBasinsAndTransects20210623.nc (Needed for variable msftmz: Ocean Meridional Overturning Mass Streamfunction) -e3sm_to_cmip -s --realm Omon --var-list areacello, fsitherm, hfds, masso, mlotst, sfdsi, sob, soga, sos, sosga, tauuo, tauvo, thetaoga, tob, tos, tosga, volo, wfo, zos, thetaoga, hfsifrazil, masscello, so, thetao, thkcello, uo, vo, volcello, wo, zhalfo --map ${e2c_path}/maps/map_EC30to60E2r2_to_cmip6_180x360_aave.20220301.nc --input-path ${model_data}/v2.mpaso_input/ --output-path ${result_dir} --user-metadata ${metadata_path} --tables-path ${e2c_path}/cmor/cmip6-cmor-tables/Tables + +# BEYOND THIS POINT, We attempt to replace "${model_data}/v2.mpaso_input/" with a directory of symlinks ("native_data"), intended to support a +# user-specified "years-per-file" (YPF) value, and to call e3sm_to_cmip on those lonks in a loop that updates the links for each YPF segment. + +ts=`date -u +%Y%m%d_%H%M%S_%6N` +runlog="e2e_e2c-${ts}.log" + +Omon_var_list="areacello, fsitherm, hfds, masso, mlotst, sfdsi, sob, soga, sos, sosga, tauuo, tauvo, thetaoga, tob, tos, tosga, volo, wfo, zos, thetaoga, hfsifrazil, masscello, so, thetao, thkcello, uo, vo, volcello, wo, zhalfo" +mapfile=${e2c_path}/maps/map_EC30to60E2r2_to_cmip6_180x360_aave.20220301.nc + +native_src=${model_data}/v2.mpaso_input/ + +in_count=`ls $native_src | wc -l` +echo "NATIVE_SOURCE_COUNT=$in_count files ($((in_count / 12)) years)" >> $runlog 2>&1 + + +real_native_src=/p/user_pub/work/E3SM/2_0/historical/LR/ocean/native/model-output/mon/ens1/v20220806 + +# Determine range of years and number of segments from the available native input (model_data). +start_year=`ls $real_native_src | grep mpaso.hist.am.timeSeriesStatsMonthly | rev | cut -f2 -d. | rev | cut -f1 -d- | head -1` +final_year=`ls $real_native_src | grep mpaso.hist.am.timeSeriesStatsMonthly | rev | cut -f2 -d. | rev | cut -f1 -d- | tail -1` +range_years=$((10#$final_year - 10#$start_year + 1)) +ypf=20 +range_segs=$((range_years/ypf)) +if [[ $((range_segs*ypf)) -lt $range_years ]]; then range_segs=$((range_segs + 1)); fi + + +native_data="native_links" +mkdir -p $native_data +rm $native_data/* + +# WORK: To begin, we need symlinks in native_data to ALL files in model_output, because the restart, namefile and region_mask files are there. + +for afile in `ls $native_src`; do + ln -s ${native_src}/$afile $native_data/$afile 2>/dev/null +done + +# for this test, remove links to these datafiles, because we will use the full published years. +for afile in `ls ${native_data}/*mpaso.hist.am.timeSeriesStatsMonthly*.nc 2>/dev/null`; do + rm -f $afile +done + +for ((segdex=0;segdex/dev/null`; do + rm -f $afile + done + + for ((yrdex=0;yrdex/dev/null + done + done + + year_init=$((10#$start_year + segdex*ypf)) + year_last=$((10#$start_year + segdex*ypf + yrdex - 1)) + ts=`date -u +%Y%m%d_%H%M%S_%6N` + echo "$ts: Calling e3sm_to_cmip for segment years $year_init to $year_last" >> $runlog + + # e3sm_to_cmip -s --realm Omon --var-list $Omon_var_list --map ${mapfile} --input-path ${native_data} --output-path ${result_dir} --user-metadata ${metadata_path} --tables-path ${tables_path} >> $runlog 2>&1 + +done + exit From e592cd71a6ab33b106a09bceb5e98e6f24ee2804 Mon Sep 17 00:00:00 2001 From: Tony Bartoletti Date: Wed, 10 Jul 2024 10:28:43 -0700 Subject: [PATCH 7/9] minor corrections to remove dryrun testing conditions --- scripts/example_end_to_end_script.sh | 110 ++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 9 deletions(-) diff --git a/scripts/example_end_to_end_script.sh b/scripts/example_end_to_end_script.sh index 8200dbfc..109328d8 100644 --- a/scripts/example_end_to_end_script.sh +++ b/scripts/example_end_to_end_script.sh @@ -37,6 +37,104 @@ map_file=${e2c_path}/maps/map_ne30pg2_to_cmip6_180x360_aave.20200201.nc tables_path=${e2c_path}/cmor/cmip6-cmor-tables/Tables/ metadata_path=${e2c_path}/user_metadata.json +# NOTE: Space is not accepted in nco var list +## ------------------------------------------------------ +## TEST CASE - atm monthly h0 +## ------------------------------------------------------ +input_path=${model_data}/v2.eam_input +flags='-7 --dfl_lvl=1 --no_cll_msr' +raw_var_list="ICEFRAC,OCNFRAC,LANDFRAC,PHIS,hyam,hybm,hyai,hybi,TREFHT,TS,PSL,PS,U10,QREFHT,PRECC,PRECL,PRECSC,PRECSL,QFLX,TAUX,TAUY,LHFLX,CLDTOT,FLDS,FLNS,FSDS,FSNS,SHFLX,CLOUD,CLDICE,TGCLDIWP,CLDLIQ,TGCLDCWP,TMQ,FLNSC,FSNTOA,FSNT,FLNT,FLUTC,FSDSC,SOLIN,FSNSC,FSUTOA,FSUTOAC,AODABS,AODVIS,AREL,TREFMNAV,TRE +FMXAV,FISCCP1_COSP,CLDTOT_ISCCP,MEANCLDALB_ISCCP,MEANPTOP_ISCCP,CLD_CAL,CLDTOT_CAL,CLDLOW_CAL,CLDMED_CAL,CLDHGH_CAL" + +# 1. atm 2D variables +#-------------------------- +cmip_var_list="pfull, phalf, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, cl, clw, cli, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin, clisccp, cltisccp, albisccp, pctis +ccp, clcalipso, cltcalipso, cllcalipso, clmcalipso, clhcalipso" +ncclimo -P eam -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} +# CMORIZE Atmosphere monthly variables: 2D and model level 3D variables (CLOUD,CLDICE,CLDLIQ) +e3sm_to_cmip -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t $tables_path -u ${metadata_path} + +# 2. atm fixed variables +#-------------------------- +raw_var_list="area,PHIS,LANDFRAC" +cmip_var_list="areacella, sftlf, orog" +ncremap --map=${map_file} -v area,PHIS,LANDFRAC -I ${input_path} -O ${rgr_dir}/fixed_vars + +# CMORIZE Atmosphere fx variables +e3sm_to_cmip --realm fx -i ${rgr_dir}/fixed_vars -o $result_dir -v areacella, sftlf, orog -t ${tables_path} -u ${metadata_path} + +## ------------------------------------------------------ +## TEST CASE - atm 3D variables +## ------------------------------------------------------ +flags='-7 --dfl_lvl=1 --no_cll_msr' +raw_var_list="Q,O3,T,U,V,Z3,RELHUM,OMEGA" +cmip_var_list="hur, hus, ta, ua, va, wap, zg, o3" +ncclimo -P eam -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir_vert} -v ${raw_var_list} -i ${input_path} ${flags} +for file in `ls ${rgr_dir_vert}` +do + ncks --rgr xtr_mth=mss_val --vrt_fl=${e2c_path}/grids/vrt_remap_plev19.nc ${rgr_dir_vert}/$file ${rgr_dir}/$file +done + +# Note --start=$start --end=$end would not work with ncremap +#ncremap -P eam -j 1 --xtr_mth=mss_val --vrt_fl=${e2c_path}/grids/vrt_remap_plev19.nc -O ${rgr_dir} -v ${raw_var_list} -I ${rgr_dir_vert} ${flags} + +# CMORIZE Atmosphere monthly plev variables +e3sm_to_cmip -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} -u ${metadata_path} + +## ------------------------------------------------------ +## TEST CASE - atm high freq daily h1+ +## ------------------------------------------------------ +input_path=${model_data}/v2.eam.h1_input +flags='-7 --dfl_lvl=1 --no_cll_msr --clm_md=hfs' +raw_var_list="TREFHTMN,TREFHTMX,PRECT,TREFHT,FLUT,QREFHT" +cmip_var_list="tasmin, tasmax, tas, huss, rlut, pr" +rgr_dir=${result_dir}/rgr_day +native_dir=${result_dir}/native_day +ncclimo -P eam -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} + +# CMORIZE Atmosphere daily variables +e3sm_to_cmip --freq day -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} -u ${metadata_path} + +## ------------------------------------------------------ +## TEST CASE - atm high freq 3hrly h1+ +## ------------------------------------------------------ +input_path=${model_data}/v2.eam.h4_input +flags='-7 --dfl_lvl=1 --no_cll_msr --clm_md=hfs' +raw_var_list="PRECT" +rgr_dir=${result_dir}/rgr_3hr +native_dir=${result_dir}/native_3hr +cmip_var_list="pr" +ncclimo -P eam -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} + +# CMORIZE Atmosphere 3hrly variables +e3sm_to_cmip --freq 3hr -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} -u ${metadata_path} + +## ------------------------------------------------------ +## TEST CASE - land monthly h0 +## ------------------------------------------------------ +input_path=${model_data}/v2.elm_input/ +flags='-7 --dfl_lvl=1 --no_cll_msr' +raw_var_list="LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO" +cmip_var_list="mrsos, mrso, mrfso, mrros, mrro, prveg, evspsblveg, evspsblsoi, tran, tsl, lai" +rgr_dir=${result_dir}/rgr_lnd +native_dir=${result_dir}/native_lnd + +# Note either include the extra variable landfrac or specify the file that has landfrac for subgrid scale mode to work. +ncclimo -P elm -j 1 --var_xtr=landfrac --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} +# Alternative ncclimo invocation +#ncclimo -P elm -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} --sgs_frc=${input_path}/v2.LR.historical_0101.elm.h0.1850-01.nc/landfrac + +#raw_var_list_elm_bgc="TOTLITC,CWDC,TOTPRODC,SOIL1C,SOIL2C,SOIL3C,^SOIL4C$,COL_FIRE_CLOSS,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR" +# CMORIZE Land Monthly variables +e3sm_to_cmip -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} -u ${metadata_path} + +# CMORIZE Sea-ice Monthly variables +# Note the input folder for mpas sea ice files requires: +# 1. Monthly mean history files: e.g.,v2.LR.historical_0101.mpassi.hist.am.timeSeriesStatsMonthly.1850-01-01.nc +# 2. A restart file for meshes: can use the mpaso restart e.g., v2.LR.historical_0101.mpaso.rst.1855-01-01_00000.nc +e3sm_to_cmip -s --realm SImon --var-list siconc, sitemptop, sisnmass, sitimefrac, siu, siv, sithick, sisnthick, simass --map ${e2c_path}/maps/map_EC30to60E2r2_to_cmip6_180x360_aave.20220301.nc --input-path ${model_data}/v2.mpassi_input/ --output-path ${result_dir} --user-metadata ${metadata_path} --tables +-path ${e2c_path}/cmor/cmip6-cmor-tables/Tables + # CMORIZE Ocean Monthly variables # Note the input folder for mpas ocean files requires: @@ -58,15 +156,9 @@ mapfile=${e2c_path}/maps/map_EC30to60E2r2_to_cmip6_180x360_aave.20220301.nc native_src=${model_data}/v2.mpaso_input/ -in_count=`ls $native_src | wc -l` -echo "NATIVE_SOURCE_COUNT=$in_count files ($((in_count / 12)) years)" >> $runlog 2>&1 - - -real_native_src=/p/user_pub/work/E3SM/2_0/historical/LR/ocean/native/model-output/mon/ens1/v20220806 - # Determine range of years and number of segments from the available native input (model_data). -start_year=`ls $real_native_src | grep mpaso.hist.am.timeSeriesStatsMonthly | rev | cut -f2 -d. | rev | cut -f1 -d- | head -1` -final_year=`ls $real_native_src | grep mpaso.hist.am.timeSeriesStatsMonthly | rev | cut -f2 -d. | rev | cut -f1 -d- | tail -1` +start_year=`ls $native_src | grep mpaso.hist.am.timeSeriesStatsMonthly | rev | cut -f2 -d. | rev | cut -f1 -d- | head -1` +final_year=`ls $native_src | grep mpaso.hist.am.timeSeriesStatsMonthly | rev | cut -f2 -d. | rev | cut -f1 -d- | tail -1` range_years=$((10#$final_year - 10#$start_year + 1)) ypf=20 range_segs=$((range_years/ypf)) @@ -105,7 +197,7 @@ for ((segdex=0;segdex/dev/null done From 706104bef42608910cfc8ecf1f179151d3aed1af Mon Sep 17 00:00:00 2001 From: Tony Bartoletti Date: Wed, 10 Jul 2024 10:38:43 -0700 Subject: [PATCH 8/9] hopefully removed odd cut/paste artifacts --- scripts/example_end_to_end_script.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/example_end_to_end_script.sh b/scripts/example_end_to_end_script.sh index 109328d8..4199e0e8 100644 --- a/scripts/example_end_to_end_script.sh +++ b/scripts/example_end_to_end_script.sh @@ -43,13 +43,11 @@ metadata_path=${e2c_path}/user_metadata.json ## ------------------------------------------------------ input_path=${model_data}/v2.eam_input flags='-7 --dfl_lvl=1 --no_cll_msr' -raw_var_list="ICEFRAC,OCNFRAC,LANDFRAC,PHIS,hyam,hybm,hyai,hybi,TREFHT,TS,PSL,PS,U10,QREFHT,PRECC,PRECL,PRECSC,PRECSL,QFLX,TAUX,TAUY,LHFLX,CLDTOT,FLDS,FLNS,FSDS,FSNS,SHFLX,CLOUD,CLDICE,TGCLDIWP,CLDLIQ,TGCLDCWP,TMQ,FLNSC,FSNTOA,FSNT,FLNT,FLUTC,FSDSC,SOLIN,FSNSC,FSUTOA,FSUTOAC,AODABS,AODVIS,AREL,TREFMNAV,TRE -FMXAV,FISCCP1_COSP,CLDTOT_ISCCP,MEANCLDALB_ISCCP,MEANPTOP_ISCCP,CLD_CAL,CLDTOT_CAL,CLDLOW_CAL,CLDMED_CAL,CLDHGH_CAL" +raw_var_list="ICEFRAC,OCNFRAC,LANDFRAC,PHIS,hyam,hybm,hyai,hybi,TREFHT,TS,PSL,PS,U10,QREFHT,PRECC,PRECL,PRECSC,PRECSL,QFLX,TAUX,TAUY,LHFLX,CLDTOT,FLDS,FLNS,FSDS,FSNS,SHFLX,CLOUD,CLDICE,TGCLDIWP,CLDLIQ,TGCLDCWP,TMQ,FLNSC,FSNTOA,FSNT,FLNT,FLUTC,FSDSC,SOLIN,FSNSC,FSUTOA,FSUTOAC,AODABS,AODVIS,AREL,TREFMNAV,TREFMXAV,FISCCP1_COSP,CLDTOT_ISCCP,MEANCLDALB_ISCCP,MEANPTOP_ISCCP,CLD_CAL,CLDTOT_CAL,CLDLOW_CAL,CLDMED_CAL,CLDHGH_CAL" # 1. atm 2D variables #-------------------------- -cmip_var_list="pfull, phalf, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, cl, clw, cli, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin, clisccp, cltisccp, albisccp, pctis -ccp, clcalipso, cltcalipso, cllcalipso, clmcalipso, clhcalipso" +cmip_var_list="pfull, phalf, tas, ts, psl, ps, sfcWind, huss, pr, prc, prsn, evspsbl, tauu, tauv, hfls, clt, rlds, rlus, rsds, rsus, hfss, cl, clw, cli, clivi, clwvi, prw, rldscs, rlut, rlutcs, rsdt, rsuscs, rsut, rsutcs, rtmt, abs550aer, od550aer, rsdscs, tasmax, tasmin, clisccp, cltisccp, albisccp, pctisccp, clcalipso, cltcalipso, cllcalipso, clmcalipso, clhcalipso" ncclimo -P eam -j 1 --map=${map_file} --start=$start --end=$end --ypf=$ypf --split -c $caseid -o ${native_dir} -O ${rgr_dir} -v ${raw_var_list} -i ${input_path} ${flags} # CMORIZE Atmosphere monthly variables: 2D and model level 3D variables (CLOUD,CLDICE,CLDLIQ) e3sm_to_cmip -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t $tables_path -u ${metadata_path} @@ -132,8 +130,7 @@ e3sm_to_cmip -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} # Note the input folder for mpas sea ice files requires: # 1. Monthly mean history files: e.g.,v2.LR.historical_0101.mpassi.hist.am.timeSeriesStatsMonthly.1850-01-01.nc # 2. A restart file for meshes: can use the mpaso restart e.g., v2.LR.historical_0101.mpaso.rst.1855-01-01_00000.nc -e3sm_to_cmip -s --realm SImon --var-list siconc, sitemptop, sisnmass, sitimefrac, siu, siv, sithick, sisnthick, simass --map ${e2c_path}/maps/map_EC30to60E2r2_to_cmip6_180x360_aave.20220301.nc --input-path ${model_data}/v2.mpassi_input/ --output-path ${result_dir} --user-metadata ${metadata_path} --tables --path ${e2c_path}/cmor/cmip6-cmor-tables/Tables +e3sm_to_cmip -s --realm SImon --var-list siconc, sitemptop, sisnmass, sitimefrac, siu, siv, sithick, sisnthick, simass --map ${e2c_path}/maps/map_EC30to60E2r2_to_cmip6_180x360_aave.20220301.nc --input-path ${model_data}/v2.mpassi_input/ --output-path ${result_dir} --user-metadata ${metadata_path} --tables-path ${e2c_path}/cmor/cmip6-cmor-tables/Tables # CMORIZE Ocean Monthly variables From 99aa8bcca6f36e3f53186742bcd36b8bf46dd4f4 Mon Sep 17 00:00:00 2001 From: Tony Bartoletti Date: Wed, 10 Jul 2024 10:42:55 -0700 Subject: [PATCH 9/9] hopefully removed odd cut/paste artifacts again --- scripts/example_end_to_end_script.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/example_end_to_end_script.sh b/scripts/example_end_to_end_script.sh index 4199e0e8..fc367efb 100644 --- a/scripts/example_end_to_end_script.sh +++ b/scripts/example_end_to_end_script.sh @@ -132,7 +132,6 @@ e3sm_to_cmip -i ${rgr_dir} -o $result_dir -v ${cmip_var_list} -t ${tables_path} # 2. A restart file for meshes: can use the mpaso restart e.g., v2.LR.historical_0101.mpaso.rst.1855-01-01_00000.nc e3sm_to_cmip -s --realm SImon --var-list siconc, sitemptop, sisnmass, sitimefrac, siu, siv, sithick, sisnthick, simass --map ${e2c_path}/maps/map_EC30to60E2r2_to_cmip6_180x360_aave.20220301.nc --input-path ${model_data}/v2.mpassi_input/ --output-path ${result_dir} --user-metadata ${metadata_path} --tables-path ${e2c_path}/cmor/cmip6-cmor-tables/Tables - # CMORIZE Ocean Monthly variables # Note the input folder for mpas ocean files requires: # 1. History files: e.g.,v2.LR.historical_0101.mpaso.hist.am.timeSeriesStatsMonthly.1850-01-01.nc @@ -205,7 +204,7 @@ for ((segdex=0;segdex> $runlog - # e3sm_to_cmip -s --realm Omon --var-list $Omon_var_list --map ${mapfile} --input-path ${native_data} --output-path ${result_dir} --user-metadata ${metadata_path} --tables-path ${tables_path} >> $runlog 2>&1 + e3sm_to_cmip -s --realm Omon --var-list $Omon_var_list --map ${mapfile} --input-path ${native_data} --output-path ${result_dir} --user-metadata ${metadata_path} --tables-path ${tables_path} >> $runlog 2>&1 done