From 68a935236f3741f0eef99d55b0c4125d4b46990c Mon Sep 17 00:00:00 2001 From: Kristin Chang Date: Wed, 29 Oct 2025 14:57:48 -0700 Subject: [PATCH 1/5] Adding cdp to pmp utils --- conda-env/ci.yml | 1 - conda-env/dev.yml | 1 - .../cloud_feedback/lib/argparse_functions.py | 2 +- .../scripts/compositeDiurnalStatistics.py | 4 +- .../diurnal/scripts/computeStdOfDailyMeans.py | 4 +- .../diurnal/scripts/std_of_dailymeans.py | 4 +- .../diurnal/scripts/std_of_hourlyvalues.py | 4 +- .../scripts/std_of_meandiurnalcycle.py | 4 +- .../drcdm/lib/create_drcdm_parser.py | 2 +- pcmdi_metrics/enso/lib/enso_lib.py | 2 +- pcmdi_metrics/io/__init__.py | 1 + pcmdi_metrics/io/base.py | 5 +- pcmdi_metrics/io/cdp_io.py | 16 + .../lib/create_mean_climate_parser.py | 2 +- .../pcmdi_compute_climatologies.py | 2 +- pcmdi_metrics/mjo/mjo_metrics_driver.py | 2 +- .../monsoon_sperber/driver_monsoon_sperber.py | 2 +- .../monsoon_wang/lib/argparse_functions.py | 2 +- .../precip_distribution_driver.py | 2 +- ...variability_across_timescales_PS_driver.py | 2 +- pcmdi_metrics/sea_ice/lib/sea_ice_parser.py | 2 +- pcmdi_metrics/utils/__init__.py | 4 + pcmdi_metrics/utils/cdp_parameter.py | 69 ++ pcmdi_metrics/utils/cdp_parser.py | 682 ++++++++++++++++++ pcmdi_metrics/utils/cdp_run.py | 65 ++ .../lib => utils}/pmp_parameter.py | 2 +- .../{mean_climate/lib => utils}/pmp_parser.py | 8 +- .../variability_modes_driver.py | 2 +- 28 files changed, 866 insertions(+), 32 deletions(-) create mode 100644 pcmdi_metrics/io/cdp_io.py create mode 100644 pcmdi_metrics/utils/cdp_parameter.py create mode 100644 pcmdi_metrics/utils/cdp_parser.py create mode 100644 pcmdi_metrics/utils/cdp_run.py rename pcmdi_metrics/{mean_climate/lib => utils}/pmp_parameter.py (99%) rename pcmdi_metrics/{mean_climate/lib => utils}/pmp_parser.py (83%) diff --git a/conda-env/ci.yml b/conda-env/ci.yml index 7f6c3858c..66b6f3c54 100644 --- a/conda-env/ci.yml +++ b/conda-env/ci.yml @@ -13,7 +13,6 @@ dependencies: - numpy >=2.0.0,<3 - cartopy >=0.22.0 - matplotlib >=3.7.1 - - cdp >=1.7.0 - eofs >=2.0.0 - seaborn >=0.12.2 #- enso_metrics >=1.1.5 diff --git a/conda-env/dev.yml b/conda-env/dev.yml index 3d54c630d..f3d7a60be 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -13,7 +13,6 @@ dependencies: - numpy >=2.0.0,<3 - cartopy >=0.22.0 - matplotlib >=3.7.1 - - cdp >=1.7.0 - eofs >=2.0.0 - seaborn >=0.12.2 #- enso_metrics >=1.1.5 diff --git a/pcmdi_metrics/cloud_feedback/lib/argparse_functions.py b/pcmdi_metrics/cloud_feedback/lib/argparse_functions.py index a35d20989..12fa63b7a 100644 --- a/pcmdi_metrics/cloud_feedback/lib/argparse_functions.py +++ b/pcmdi_metrics/cloud_feedback/lib/argparse_functions.py @@ -1,6 +1,6 @@ import argparse -from cdp.cdp_parser import CDPParser +from pcmdi_metrics.utils.cdp_parser import CDPParser def AddParserArgument(): diff --git a/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py b/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py index a801b1c8f..152591048 100755 --- a/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py +++ b/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py @@ -24,7 +24,7 @@ import multiprocessing as mp import os -import cdp +from pcmdi_metrics.utils import cdp_run import cftime import numpy as np import xarray as xr @@ -252,7 +252,7 @@ def compute(params): print("FILES:", fileList) params = [INPUT(args, name, template) for name in fileList] print("PARAMS:", params) - cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers) + cdp_run.multiprocess(compute, params, num_workers=args.num_workers) def add_one_month(t): diff --git a/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py b/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py index c51718670..6e4934f4a 100755 --- a/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py +++ b/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py @@ -16,7 +16,7 @@ import multiprocessing as mp import os -import cdp +from pcmdi_metrics.utils import cdp_run import cftime import xarray as xr @@ -195,7 +195,7 @@ def compute(params, debug=False): params = [INPUT(args, name, template) for name in fileList] print("PARAMS:", params) - cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers) + cdp_run.multiprocess(compute, params, num_workers=args.num_workers) def add_one_month(t): diff --git a/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py b/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py index 008598b2d..ea31e31f8 100755 --- a/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py +++ b/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py @@ -24,7 +24,7 @@ import multiprocessing as mp import os -import cdp +from pcmdi_metrics.utils import cdp_run import numpy as np import pcmdi_metrics @@ -167,7 +167,7 @@ def compute(param): params = [INPUT(args, name, template) for name in files] print("PARAMS:", params) - results = cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers) + results = cdp_run.multiprocess(compute, params, num_workers=args.num_workers) for r in results: m, region, res = r diff --git a/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py b/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py index c6768bc44..945c6618b 100755 --- a/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py +++ b/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py @@ -21,7 +21,7 @@ import multiprocessing as mp import os -import cdp +from pcmdi_metrics.utils import cdp_run import pcmdi_metrics from pcmdi_metrics import resources @@ -172,7 +172,7 @@ def compute(param): params = [INPUT(args, name, template) for name in files] print("PARAMS:", params) - results = cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers) + results = cdp_run.multiprocess(compute, params, num_workers=args.num_workers) for r in results: m, region, res = r diff --git a/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py b/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py index cd0c3de14..a626b5af2 100755 --- a/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py +++ b/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py @@ -19,7 +19,7 @@ import multiprocessing as mp import os -import cdp +from pcmdi_metrics.utils import cdp_run import pcmdi_metrics from pcmdi_metrics import resources @@ -182,7 +182,7 @@ def compute(param): params = [INPUT(args, name, template) for name in files] print("PARAMS:", params) - results = cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers) + results = cdp_run.multiprocess(compute, params, num_workers=args.num_workers) for r in results: m, region, res = r diff --git a/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py b/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py index 4ea851140..f35536c72 100644 --- a/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py +++ b/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from pcmdi_metrics.mean_climate.lib import pmp_parser +from pcmdi_metrics.utils import pmp_parser def create_extremes_parser(): diff --git a/pcmdi_metrics/enso/lib/enso_lib.py b/pcmdi_metrics/enso/lib/enso_lib.py index 38452d6f0..06e781023 100755 --- a/pcmdi_metrics/enso/lib/enso_lib.py +++ b/pcmdi_metrics/enso/lib/enso_lib.py @@ -7,7 +7,7 @@ from collections import defaultdict import pcmdi_metrics -from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser +from pcmdi_metrics.utils.pmp_parser import PMPParser def AddParserArgument(): diff --git a/pcmdi_metrics/io/__init__.py b/pcmdi_metrics/io/__init__.py index eddc7d8de..694d6d9d8 100644 --- a/pcmdi_metrics/io/__init__.py +++ b/pcmdi_metrics/io/__init__.py @@ -2,6 +2,7 @@ from .xcdat_openxml import xcdat_open # noqa # isort:skip from .string_constructor import StringConstructor, fill_template # noqa # isort:skip from . import base # noqa +from . import cdp_io # noqa from .xcdat_dataset_io import ( # noqa # isort:skip da_to_ds, diff --git a/pcmdi_metrics/io/base.py b/pcmdi_metrics/io/base.py index 50b5b55a8..9d133130f 100755 --- a/pcmdi_metrics/io/base.py +++ b/pcmdi_metrics/io/base.py @@ -11,13 +11,12 @@ from datetime import datetime from subprocess import PIPE, Popen -import cdp.cdp_io import numpy import requests import pcmdi_metrics from pcmdi_metrics import LOG_LEVEL -from pcmdi_metrics.io import StringConstructor +from pcmdi_metrics.io import cdp_io, StringConstructor logging.getLogger("pcmdi_metrics").setLevel(LOG_LEVEL) # set up to log errors @@ -264,7 +263,7 @@ def alphanum(key): # ---------- -class Base(cdp.cdp_io.CDPIO, StringConstructor): +class Base(cdp_io.CDPIO, StringConstructor): def __init__(self, root, file_template, file_mask_template=None): StringConstructor.__init__(self, root + "/" + file_template) self.root = root diff --git a/pcmdi_metrics/io/cdp_io.py b/pcmdi_metrics/io/cdp_io.py new file mode 100644 index 000000000..1bd17bcb7 --- /dev/null +++ b/pcmdi_metrics/io/cdp_io.py @@ -0,0 +1,16 @@ +from __future__ import print_function + +import abc +from six import with_metaclass + + +class CDPIO(with_metaclass(abc.ABCMeta, object)): + @abc.abstractmethod + def read(self): + """Read a file.""" + raise NotImplementedError() + + @abc.abstractmethod + def write(self): + """Write a file.""" + raise NotImplementedError() \ No newline at end of file diff --git a/pcmdi_metrics/mean_climate/lib/create_mean_climate_parser.py b/pcmdi_metrics/mean_climate/lib/create_mean_climate_parser.py index f5f97a7cc..a0c04e977 100644 --- a/pcmdi_metrics/mean_climate/lib/create_mean_climate_parser.py +++ b/pcmdi_metrics/mean_climate/lib/create_mean_climate_parser.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import ast -from pcmdi_metrics.mean_climate.lib import pmp_parser +from pcmdi_metrics.utils import pmp_parser def create_mean_climate_parser(): diff --git a/pcmdi_metrics/mean_climate/pcmdi_compute_climatologies.py b/pcmdi_metrics/mean_climate/pcmdi_compute_climatologies.py index bc9fa6bc8..6865a01c0 100755 --- a/pcmdi_metrics/mean_climate/pcmdi_compute_climatologies.py +++ b/pcmdi_metrics/mean_climate/pcmdi_compute_climatologies.py @@ -4,7 +4,7 @@ from pcmdi_metrics.io import StringConstructor from pcmdi_metrics.mean_climate.lib import calculate_climatology -from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPMetricsParser +from pcmdi_metrics.utils.pmp_parser import PMPMetricsParser ver = datetime.datetime.now().strftime("v%Y%m%d") diff --git a/pcmdi_metrics/mjo/mjo_metrics_driver.py b/pcmdi_metrics/mjo/mjo_metrics_driver.py index d34cffb30..ae4f03072 100755 --- a/pcmdi_metrics/mjo/mjo_metrics_driver.py +++ b/pcmdi_metrics/mjo/mjo_metrics_driver.py @@ -42,7 +42,7 @@ from shutil import copyfile import pcmdi_metrics -from pcmdi_metrics.mean_climate.lib import pmp_parser +from pcmdi_metrics.utils import pmp_parser from pcmdi_metrics.mjo.lib import ( AddParserArgument, YearCheck, diff --git a/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py b/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py index ff37a883c..6061c443f 100644 --- a/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py +++ b/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py @@ -54,7 +54,7 @@ from pcmdi_metrics.io import load_regions_specs, region_subset, xcdat_open from pcmdi_metrics.io.base import Base -from pcmdi_metrics.mean_climate.lib import pmp_parser +from pcmdi_metrics.utils import pmp_parser from pcmdi_metrics.monsoon_sperber.lib import ( AddParserArgument, YearCheck, diff --git a/pcmdi_metrics/monsoon_wang/lib/argparse_functions.py b/pcmdi_metrics/monsoon_wang/lib/argparse_functions.py index eb28b515d..b4a3d0dce 100644 --- a/pcmdi_metrics/monsoon_wang/lib/argparse_functions.py +++ b/pcmdi_metrics/monsoon_wang/lib/argparse_functions.py @@ -1,4 +1,4 @@ -from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser +from pcmdi_metrics.utils.pmp_parser import PMPParser def create_monsoon_wang_parser(): diff --git a/pcmdi_metrics/precip_distribution/precip_distribution_driver.py b/pcmdi_metrics/precip_distribution/precip_distribution_driver.py index 3189f3f11..c911ad725 100644 --- a/pcmdi_metrics/precip_distribution/precip_distribution_driver.py +++ b/pcmdi_metrics/precip_distribution/precip_distribution_driver.py @@ -7,7 +7,7 @@ import xarray as xr from pcmdi_metrics.io import StringConstructor, get_calendar, xcdat_open -from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser +from pcmdi_metrics.utils.pmp_parser import PMPParser from pcmdi_metrics.precip_distribution.lib import ( AddParserArgument, Regrid_xr, diff --git a/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py b/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py index 5a9b4cdae..a4abe1657 100644 --- a/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py +++ b/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py @@ -2,7 +2,7 @@ import glob import os -from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser +from pcmdi_metrics.utils.pmp_parser import PMPParser from pcmdi_metrics.precip_variability.lib import ( AddParserArgument, precip_variability_across_timescale, diff --git a/pcmdi_metrics/sea_ice/lib/sea_ice_parser.py b/pcmdi_metrics/sea_ice/lib/sea_ice_parser.py index 3ba642d99..5cd70f64a 100644 --- a/pcmdi_metrics/sea_ice/lib/sea_ice_parser.py +++ b/pcmdi_metrics/sea_ice/lib/sea_ice_parser.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from pcmdi_metrics.mean_climate.lib import pmp_parser +from pcmdi_metrics.utils import pmp_parser def create_sea_ice_parser(): diff --git a/pcmdi_metrics/utils/__init__.py b/pcmdi_metrics/utils/__init__.py index 8b2a5c3d8..04128c967 100644 --- a/pcmdi_metrics/utils/__init__.py +++ b/pcmdi_metrics/utils/__init__.py @@ -31,3 +31,7 @@ from .string_constructor import StringConstructor, fill_template from .tree_dict import tree from .xr_to_cdms2 import cdms2_to_xarray, xarray_to_cdms2 +from .cdp_parser import CDPParser +from .cdp_parameter import CDPParameter +from .pmp_parser import PMPParser, PMPMetricsParser +from .pmp_parameter import PMPParameter, PMPMetricsParameter diff --git a/pcmdi_metrics/utils/cdp_parameter.py b/pcmdi_metrics/utils/cdp_parameter.py new file mode 100644 index 000000000..7b91ab6a5 --- /dev/null +++ b/pcmdi_metrics/utils/cdp_parameter.py @@ -0,0 +1,69 @@ +from __future__ import print_function + +import abc +import importlib +import sys +import os +import copy +import types +from six import with_metaclass + + +class CDPParameter(object): + def __add__(self, other): + """ + Deepcopy any attribute of `other` into self. + """ + # First make a deepcopy of the current object. + duplicate = copy.deepcopy(self) + + for attr in dir(other): + # Ignore any of the hidden attributes. + if attr.startswith('_') or \ + isinstance(getattr(other, attr), types.MethodType): + continue + + val = copy.deepcopy(getattr(other, attr)) + setattr(duplicate, attr, val) + + return duplicate + + def check_values(self): + """ + Check that all of the variables in + this parameter file are valid. + """ + pass + + def load_parameter_from_py(self, parameter_file_path): + """ + Initialize a parameter object from a Python script. + """ + parameter_as_module = \ + self.import_user_parameter_file_as_module(parameter_file_path) + self.load_parameters_from_module(parameter_as_module) + + def import_user_parameter_file_as_module(self, parameter_file_path): + if not os.path.isfile(parameter_file_path): + raise IOError('Parameter file %s not found.' % parameter_file_path) + + path_to_module = os.path.split(parameter_file_path)[0] + module_name = os.path.split(parameter_file_path)[1] + if module_name.count('.') > 1: + raise ValueError("Filename cannot contain '.' outside extension.") + if '.' in module_name: + module_name = module_name.split('.')[0] + + sys.path.insert(0, path_to_module) + return importlib.import_module(module_name) + + def load_parameters_from_module(self, parameter_as_module): + user_defined_parameters = [] + for user_parameter in dir(parameter_as_module): + if not user_parameter.startswith('__'): + user_defined_parameters.append(user_parameter) + + # Initialize the variables in this parameter, so the driver can + # access them as if they were defined regularly. + for p in user_defined_parameters: + self.__dict__[p] = getattr(parameter_as_module, p) \ No newline at end of file diff --git a/pcmdi_metrics/utils/cdp_parser.py b/pcmdi_metrics/utils/cdp_parser.py new file mode 100644 index 000000000..d2c5b1aa4 --- /dev/null +++ b/pcmdi_metrics/utils/cdp_parser.py @@ -0,0 +1,682 @@ +from __future__ import print_function + +import sys +import argparse +import json +import yaml +import warnings +import itertools +import collections +import copy +import random +import hashlib +import types + +if sys.version_info[0] >= 3: + import configparser + from io import StringIO +else: + import ConfigParser as configparser + from StringIO import StringIO + + +class CDPParser(argparse.ArgumentParser): + def __init__(self, parameter_cls=None, default_args_file=[], + formatter_class=argparse.ArgumentDefaultsHelpFormatter, *args, **kwargs): + # conflict_handler='resolve' lets new args override older ones + self.__default_args = [] + super(CDPParser, self).__init__(conflict_handler='resolve', + formatter_class=formatter_class, + *args, **kwargs) + self.load_default_args(default_args_file) + self.__args_namespace = None + self.__parameter_cls = parameter_cls + + if not self.__parameter_cls: + from cdp.cdp_parameter import CDPParameter + self.__parameter_cls = CDPParameter + + def parse_args(self, args=None, namespace=None): + """ + Overwrites default ArgumentParser.parse_args(). + We need to save the command used to run the parser, which is args or sys.argv. + This is because the command used is not always sys.argv. + """ + self.cmd_used = sys.argv if not args else args + return super(CDPParser, self).parse_args(args, namespace) + + def view_args(self): + """" + Returns the args namespace. + """ + self._parse_arguments() + return self.__args_namespace + + def _was_command_used(self, cmdline_arg): + """ + Returns True if the cmdline_arg was used to + run the script that has this parser. + """ + # self.cmd_used is like: ['something.py', '-p', 'test.py', '--s1', 'something'] + for cmd in self.cmd_used: + # Sometimes, a command is run with '=': 'driver.py --something=this' + for c in cmd.split('='): + if cmdline_arg == c: + return True + return False + + def _is_arg_default_value(self, arg): + """ + Look at the command used for this parser (ex: test.py -s something --s1 something1) + and if arg wasn't used, then it's a default value. + """ + # Each cmdline_arg is either '-*' or '--*'. + for cmdline_arg in self._option_string_actions: + if arg == self._option_string_actions[cmdline_arg].dest and self._was_command_used(cmdline_arg): + return False + return True + + @staticmethod + def check_values_of_params(parameters): + """ + Given a list of parameters, call the check_values() + function of all of them. + """ + for p in parameters: + p.check_values() + + def _get_default_from_cmdline(self, parameters): + """ + Get the default values from the command line and insert it into the parameters object, + but only if that parameter is NOT already defined. + """ + for arg_name, arg_value in vars(self.__args_namespace).items(): + if self._is_arg_default_value(arg_name) and not hasattr(parameters, arg_name): + setattr(parameters, arg_name, arg_value) + + def _parse_arguments(self): + """ + Parse the command line arguments while checking for the user's arguments. + """ + if self.__args_namespace is None: + self.__args_namespace = self.parse_args() + + def get_orig_parameters(self, check_values=False, argparse_vals_only=True): + """ + Returns the parameters created by -p. If -p wasn't used, returns None. + """ + self._parse_arguments() + + if not self.__args_namespace.parameters: + return None + + parameter = self.__parameter_cls() + + # Remove all of the variables. + parameter.__dict__.clear() + + # if self.__args_namespace.parameters is not None: + parameter.load_parameter_from_py( + self.__args_namespace.parameters) + + if check_values: + parameter.check_values() + if argparse_vals_only: + self._only_cmdline_args(parameter) + + return parameter + + def get_parameters_from_json(self, json_file, check_values=False, argparse_vals_only=True): + """ + Given a json file, return the parameters from it. + """ + with open(json_file) as f: + json_data = json.loads(f.read()) + + parameters = [] + for key in json_data: + for single_run in json_data[key]: + p = self.__parameter_cls() + + # Remove all of the variables. + p.__dict__.clear() + + for attr_name in single_run: + setattr(p, attr_name, single_run[attr_name]) + + if check_values: + p.check_values() + if argparse_vals_only: + self._only_cmdline_args(p) + + parameters.append(p) + + return parameters + + def _create_cfg_hash_titles(self, cfg_file): + """ + Given a path to a cfg file, for any title '[#]', create a hash of it's contents + and change the title to that. Then return the StringIO object. + """ + lines = [] + with open(cfg_file) as f: + lines = f.readlines() + + h_sha256 = hashlib.sha256() + i = 0 + + while i < len(lines): + if lines[i] in ['[#]\n', '[#]']: + replace_idx = i + str_list = [] + i += 1 + while i < len(lines) and not lines[i].startswith('['): + str_list.append(lines[i]) + i += 1 + str_list.append(str(random.random())) # Randomize the hash even more. + h_sha256.update(''.join(str_list).encode()) + lines[replace_idx] = '[{}]'.format(h_sha256.hexdigest()) + else: + i += 1 + return StringIO('\n'.join(lines)) + + def get_parameters_from_cfg(self, cfg_file, check_values=False, argparse_vals_only=True): + """ + Given a cfg file, return the parameters from it. + """ + parameters = [] + + cfg_file_obj = self._create_cfg_hash_titles(cfg_file) + kwargs = {'strict': False} if sys.version_info[0] >= 3 else {} # 'strict' keyword doesn't work in Python 2. + config = configparser.ConfigParser(**kwargs) # Allow for two lines to be the same. + config.readfp(cfg_file_obj) + + for section in config.sections(): + p = self.__parameter_cls() + + # Remove all of the variables. + p.__dict__.clear() + + for k, v in config.items(section): + v = yaml.safe_load(v) + setattr(p, k, v) + + if check_values: + p.check_values() + if argparse_vals_only: + self._only_cmdline_args(p) + + parameters.append(p) + + return parameters + + def get_other_parameters(self, files_to_open=[], check_values=False, argparse_vals_only=True): + """ + Returns the parameters created by -d. If files_to_open is defined, + then use the path specified instead of -d. + """ + parameters = [] + + self._parse_arguments() + + if files_to_open == []: + files_to_open = self.__args_namespace.other_parameters + + if files_to_open is not None: + for diags_file in files_to_open: + if '.json' in diags_file: + params = self.get_parameters_from_json(diags_file, check_values, argparse_vals_only) + elif '.cfg' in diags_file: + params = self.get_parameters_from_cfg(diags_file, check_values, argparse_vals_only) + else: + raise RuntimeError( + 'The parameters input file must be either a .json or .cfg file') + + for p in params: + parameters.append(p) + + return parameters + + def _were_cmdline_args_used(self): + """ + Checks that other parameters, besides '-p' or '-d', were used. + """ + for cmd in self.cmd_used: + if cmd.startswith('-') and cmd not in ['-p', '--parameters', '-d', '--diags']: + return True + return False + + def _overwrite_parameters_with_cmdline_args(self, parameters): + """ + Add the command line parameters used to the parameter object. + """ + for arg_name, arg_value in vars(self.__args_namespace).items(): + if not self._is_arg_default_value(arg_name): + setattr(parameters, arg_name, arg_value) + + def get_cmdline_parameters(self, check_values=False, argparse_vals_only=True): + """ + Use the other command line args besides -p and -d to create a single parameters object. + """ + self._parse_arguments() + + if not self._were_cmdline_args_used(): + return None + + parameter = self.__parameter_cls() + + # Remove all of the variables + parameter.__dict__.clear() + + self._overwrite_parameters_with_cmdline_args(parameter) + + if check_values: + parameter.check_values() + if argparse_vals_only: + self._only_cmdline_args(parameter) + + return parameter + + def _only_cmdline_args(self, parameter): + """ + Remove all parameters except those that are + usable by via command line arguments. + """ + acceptable_args = vars(self.view_args()) + current_args = vars(parameter) + + params_to_del = [a for a in current_args if a not in acceptable_args] + + for param in params_to_del: + delattr(parameter, param) + + def add_default_values(self, parameter, default_vars=False, cmd_default_vars=False): + """ + Add the default values to the parameter. + These can come from the default values defined in the Parameter class, + or the `default` option defined in ArgumentParser.add_argument(). + """ + # Add the command line default parameters first. + if cmd_default_vars: + for arg_name, arg_value in vars(self.__args_namespace).items(): + if arg_name in parameter.__dict__ or not self._is_arg_default_value(arg_name): + continue + # Only add the default values, that aren't already in parameter. + setattr(parameter, arg_name, arg_value) + + # Then add the defaults defined in the Parameter class. + if default_vars: + for arg_name, arg_value in vars(self.__parameter_cls()).items(): + if arg_name in parameter.__dict__: + continue + setattr(parameter, arg_name, arg_value) + + def _get_selectors(self, cmdline_parameters=None, orig_parameters=None, other_parameters=None): + """ + Look through the cmdline_parameters, orig_parameters, and other_parameters + in that order for the selectors used. + If not defined in any of them, use the default one in the class. + """ + if hasattr(cmdline_parameters, 'selectors') and cmdline_parameters.selectors is not None: + return cmdline_parameters.selectors + elif hasattr(orig_parameters, 'selectors') and orig_parameters.selectors is not None: + return orig_parameters.selectors + elif hasattr(other_parameters, 'selectors') and other_parameters.selectors is not None: + return other_parameters.selectors + else: + # If the parameter class has selectors, try to add that in. + param = self.__parameter_cls() + if hasattr(param, 'selectors'): + return param.selectors + # None of the passed in parameters have a selector and neither the main_parameter + # nor the parameter class has selectors, so return an empty list. + return [] + + def combine_params(self, cmdline_parameters=None, orig_parameters=None, other_parameters=None, vars_to_ignore=[], default_vars=False, cmd_default_vars=False): + """ + Combine cmdline_params (-* or --*), orig_parameters (-p), and other_parameters (-d), + while ignoring any parameters listed in the 'selectors' parameter. + Add any default arguments here as well. + """ + if other_parameters: + for parameters in other_parameters: + self.add_default_values(parameters, default_vars, cmd_default_vars) + + # orig_parameters args take precedence over other_parameters. + if orig_parameters: + for var in orig_parameters.__dict__: + if var not in vars_to_ignore: + parameters.__dict__[var] = orig_parameters.__dict__[var] + + # cmd_line args take the final precedence. + if cmdline_parameters: + for var in cmdline_parameters.__dict__: + if var not in vars_to_ignore: + parameters.__dict__[var] = cmdline_parameters.__dict__[var] + + else: + # Just combine cmdline_params with orig_params. + if orig_parameters and cmdline_parameters: + self.add_default_values(orig_parameters, default_vars, cmd_default_vars) + + for var in cmdline_parameters.__dict__: + # if var not in vars_to_ignore and self._was_command_used(var): + if var not in vars_to_ignore: + # Only add it if it was not in param and was passed from cmd line. + orig_parameters.__dict__[var] = cmdline_parameters.__dict__[var] + + elif orig_parameters: + self.add_default_values(orig_parameters, default_vars, cmd_default_vars) + elif cmdline_parameters: + self.add_default_values(cmdline_parameters, default_vars, cmd_default_vars) + + def combine_orig_and_other_params(self, orig_parameters, other_parameters): + """ + Combine orig_parameters with all of the other_parameters. + """ + print('Depreciation warning: please use combine_params() instead') + self.combine_params(None, orig_parameters, other_parameters) + + def granulate(self, parameters): + """ + Given a list of parameters objects, for each parameters with a `granulate` attribute, + create multiple parameters objects for each result in the Cartesian product of `granulate`. + """ + final_parameters = [] + for param in parameters: + if not hasattr(param, 'granulate') or (hasattr(param, 'granulate') and not param.granulate): + final_parameters.append(param) + continue + + # Remove any attrs that are modules from the param object. + # These cause an error when copy.deepcopy(param) is used. + attrs = vars(param).items() + modules_in_param = [] + for var_name, var_value in attrs: + if isinstance(var_value, types.ModuleType): + modules_in_param.append(var_name) + for module in modules_in_param: + delattr(param, module) + + # Granulate param. + vars_to_granulate = param.granulate # Ex: ['seasons', 'plevs'] + # Check that all of the vars_to_granulate are iterables. + # Ex: {'season': ['ANN', 'DJF', 'MAM'], 'plevs': [850.0, 250.0]} + vals_to_granulate = collections.OrderedDict() + for v in vars_to_granulate: + if not hasattr(param, v): + raise RuntimeError("Parameters object has no attribute '{}' to granulate.".format(v)) + param_v = getattr(param, v) + if not isinstance(param_v, collections.Iterable): + raise RuntimeError("Granulate option '{}' is not an iterable.".format(v)) + if param_v: # Ignore []. + vals_to_granulate[v] = param_v + + # Ex: [('ANN', 850.0), ('ANN', 250.0), ('DJF', 850.0), ('DJF', 250.0), ...] + granulate_values = list(itertools.product(*vals_to_granulate.values())) + for g_vals in granulate_values: + p = copy.deepcopy(param) + for i, g_val in enumerate(g_vals): + key_at_index_i = list(vals_to_granulate.keys())[i] + # Make sure to insert a list with one element, + # which is why we have [g_val]. + setattr(p, key_at_index_i, [g_val]) + final_parameters.append(p) + + return final_parameters + + def select(self, main_parameters, parameters): + """ + Given a list of parameters (parameters), only return those from this list + whose 'selector' parameters are a subset of the 'selector' parameters of main_parameters. + """ + def is_subset(param1, param2): + """ + Check if param1 is a subset of param2. + These are any Python objects. + """ + if not isinstance(param1, list): + param1 = [param1] + if not isinstance(param2, list): + param2 = [param2] + + return set(param1).issubset(set(param2)) + + # Can't select from None. + if not main_parameters: + return parameters + + selectors = self._get_selectors(None, main_parameters, parameters) + + final_parameters = [] + + for param in parameters: + if all(is_subset(getattr(param, select_parameter), + getattr(main_parameters, select_parameter)) + for select_parameter in selectors): + final_parameters.append(param) + + return final_parameters + + + def _get_alias(self, param): + """ + For a single parameter, get the aliases of it. + """ + # Parameters can start with either '-' or '--'. + param = '--{}'.format(param) + if param not in self._option_string_actions: + param = '-{}'.format(param) + if param not in self._option_string_actions: + return [] + + # Ex: If param is 'parameters', then we get ['-p', '--parameters']. + aliases = self._option_string_actions[param].option_strings + + return [a.replace('-', '') for a in aliases] + + def add_aliases(self, parameters): + """ + For each of the parameters, add all of + the defined aliases as other attributes. + """ + for param in parameters: + # We need to set this info as a variable. + # Can't do: + # for param_name in vars(param) + # because we're modifying param as we iterate. + # We also need to make a copy because dicts are referenced. + param_names = copy.copy(vars(param)) + + for param_name in param_names: + param_value = getattr(param, param_name) + aliases = self._get_alias(param_name) + # Add all of the aliases for param_name to the param object. + for alias in aliases: + setattr(param, alias, param_value) + + def get_parameters(self, cmdline_parameters=None, orig_parameters=None, other_parameters=[], default_vars=True, cmd_default_vars=True, *args, **kwargs): + """ + Get the parameters based on the command line arguments and return a list of them. + """ + if not cmdline_parameters: + cmdline_parameters = self.get_cmdline_parameters(*args, **kwargs) + if not orig_parameters: + orig_parameters = self.get_orig_parameters(*args, **kwargs) + if other_parameters == []: + other_parameters = self.get_other_parameters(*args, **kwargs) + + # We don't want to add the selectors to each of the parameters. + # Because if we do, it'll select all of the parameters at the end during the selection step. + vars_to_ignore = self._get_selectors(cmdline_parameters, orig_parameters, other_parameters) + self.combine_params(cmdline_parameters, orig_parameters, other_parameters, vars_to_ignore, default_vars, cmd_default_vars) + + if other_parameters != []: + final_parameters = other_parameters + elif orig_parameters: + final_parameters = [orig_parameters] + elif cmdline_parameters: + final_parameters = [cmdline_parameters] + + # User didn't give any command line options, so create a parameter from the + # defaults of the command line argument or the Parameter class. + elif cmd_default_vars: + p = self.__parameter_cls() + for arg_name, arg_value in vars(self.__args_namespace).items(): + setattr(p, arg_name, arg_value) + final_parameters = [p] + elif default_vars: + p = self.__parameter_cls() + final_parameters = [p] + + final_parameters = self.granulate(final_parameters) + + # Only select from the -p or the command line options. + parameter = self.get_orig_parameters(*args, **kwargs) + cmdline_parameter = self.get_cmdline_parameters(*args, **kwargs) + + # Command line parameters are added to parameter. + # default_vars must be True, b/c the user excepts to select from them. + self.combine_params(cmdline_parameter, parameter, default_vars=True) + # Sometimes, one of these can be None, so get the one that's None. + parameter = parameter if parameter else cmdline_parameter + + final_parameters = self.select(parameter, final_parameters) + self.add_aliases(final_parameters) + + return final_parameters + + def get_parameter(self, warning=False, *args, **kwargs): + """ + Return the first Parameter in the list of Parameters. + """ + if warning: + print( + 'Depreciation warning: Use get_parameters() instead, which returns a list of Parameters.') + return self.get_parameters(*args, **kwargs)[0] + + def load_default_args_from_json(self, files): + """ + Take in a list of json files (or a single json file) and create the args from it. + """ + # This is needed for the loading from JSON files, + # because the type can be ast.literal_eval. + import ast + + if not isinstance(files, (list, tuple)): + files = [files] + success = None + for afile in files: + with open(afile) as json_file: + args = json.load(json_file) + for k in args.keys(): + if k[0] != "-": + continue + try: + params = args[k] + option_strings = params.pop("aliases", []) + option_strings.insert(0, k) + # Sometime we can't set a type + # like action="store_true", + # setting it to null in json file + # leads to exception in eval, + # hence not setting it. + try: + params["type"] = eval(params.pop("type", "str")) + except: + pass + self.store_default_arguments(option_strings, params) + success = True + except: + warnings.warn("Failed to load param {} from json file {}".format( + k, afile)) + + return success + + def store_default_arguments(self, options, params): + self.__default_args.insert(0,([options, params])) + + def print_available_defaults(self): + p = argparse.ArgumentParser() + for opt, param in self.__default_args: + p.add_argument(*opt, **param) + p.print_help() + + def available_defaults(self): + return [x[0] for x in self.__default_args] + + def use(self, options): + if not isinstance(options, (list, tuple)): + options = [options] + for option in options: + match = False + for opts, params in self.__default_args: + if option in opts: + match = True + break + elif option[0] != "--" and "--" + option in opts: + match = True + break + elif option[0] != "-" and "-" + option in opts: + match = True + break + if match: + self.add_argument(*opts, **params) + else: + raise RuntimeError( + "Could not match {} to any of the default arguments {}".format( + option, self.available_defaults)) + + def load_default_args(self, files=[]): + """ + Load the default arguments for the parser. + """ + if self.load_default_args_from_json(files): + return + self.add_argument( + '-p', '--parameters', + type=str, + dest='parameters', + help='Path to the user-defined parameter file.', + required=False) + self.add_argument( + '-d', '--diags', + type=str, + nargs='+', + dest='other_parameters', + default=[], + help='Path to the other user-defined parameter file.', + required=False) + self.add_argument( + '-n', '--num_workers', + type=int, + dest='num_workers', + help='Number of workers, used when running with multiprocessing or in distributed mode.', + required=False) + self.add_argument( + '--scheduler_addr', + type=str, + dest='scheduler_addr', + help='Address of the scheduler in the form of IP_ADDRESS:PORT. Used when running in distributed mode.', + required=False) + self.add_argument( + '-g', '--granulate', + type=str, + nargs='+', + dest='granulate', + help='A list of variables to granulate.', + required=False) + self.add_argument( + '--selectors', + type=str, + nargs='+', + dest='selectors', + help='A list of variables to be used to select parameters from.', + required=False) + + def add_args_and_values(self, arg_list): + """ + Used for testing. Can test args input as if they + were inputted from the command line. + """ + self.__args_namespace = self.parse_args(arg_list) \ No newline at end of file diff --git a/pcmdi_metrics/utils/cdp_run.py b/pcmdi_metrics/utils/cdp_run.py new file mode 100644 index 000000000..7e13b6de6 --- /dev/null +++ b/pcmdi_metrics/utils/cdp_run.py @@ -0,0 +1,65 @@ +from __future__ import print_function + +import dask.bag +from dask.distributed import Client + + +def serial(func, parameters): + """ + Run the function with the parameters serially. + """ + results = [] + for p in parameters: + results.append(func(p)) + return results + +def multiprocess(func, parameters, num_workers=None, context=None): + """ + Run the function with the parameters in parallel using multiprocessing. + + ``context`` is one of ``{"fork", "spawn", "forkserver"}``. For + dask<2.16.0,the default context is "fork" and for dask>=2.16.0, the default + is "spawn". + """ + bag = dask.bag.from_sequence(parameters) + + config = {'scheduler': 'processes'} + if context is not None: + config['multiprocessing.context'] = context + elif hasattr(parameters[0], 'multiprocessing_context'): + config['multiprocessing.context'] = \ + parameters[0].multiprocessing_context + + with dask.config.set(config): + if num_workers: + results = bag.map(func).compute(num_workers=num_workers) + elif hasattr(parameters[0], 'num_workers'): + results = bag.map(func).compute(num_workers=parameters[0].num_workers) + else: + # num of workers is defaulted to the number of logical processes + results = bag.map(func).compute() + + return results + +def distribute(func, parameters, scheduler_addr=None): + """ + Run the function with the parameters in parallel distributedly. + """ + try: + if scheduler_addr: + addr = scheduler_addr + elif not hasattr(parameters[0], 'scheduler_addr'): + raise RuntimeError('The parameters or distribute() need a scheduler_addr parameter.') + else: + addr = parameters[0].scheduler_addr + + client = Client(addr) + results = client.map(func, parameters) + client.gather(results) + except Exception as e: + print('Distributed run failed.') + raise e + finally: + client.close() + + return results \ No newline at end of file diff --git a/pcmdi_metrics/mean_climate/lib/pmp_parameter.py b/pcmdi_metrics/utils/pmp_parameter.py similarity index 99% rename from pcmdi_metrics/mean_climate/lib/pmp_parameter.py rename to pcmdi_metrics/utils/pmp_parameter.py index b3ee9c9d0..1d69d7183 100755 --- a/pcmdi_metrics/mean_climate/lib/pmp_parameter.py +++ b/pcmdi_metrics/utils/pmp_parameter.py @@ -1,7 +1,7 @@ import logging import os -import cdp.cdp_parameter +import pcmdi_metrics.utils as cdp from pcmdi_metrics import LOG_LEVEL from pcmdi_metrics.utils import StringConstructor diff --git a/pcmdi_metrics/mean_climate/lib/pmp_parser.py b/pcmdi_metrics/utils/pmp_parser.py similarity index 83% rename from pcmdi_metrics/mean_climate/lib/pmp_parser.py rename to pcmdi_metrics/utils/pmp_parser.py index cc4bfed9a..0cab41d4a 100644 --- a/pcmdi_metrics/mean_climate/lib/pmp_parser.py +++ b/pcmdi_metrics/utils/pmp_parser.py @@ -1,9 +1,9 @@ import os -import cdp.cdp_parser +import pcmdi_metrics.utils.cdp_parser as cdp from pcmdi_metrics import resources -from pcmdi_metrics.mean_climate.lib.pmp_parameter import ( +from pcmdi_metrics.utils.pmp_parameter import ( PMPMetricsParameter, PMPParameter, ) @@ -21,7 +21,7 @@ def path_to_default_args(): return file_path -class PMPParser(cdp.cdp_parser.CDPParser): +class PMPParser(cdp.CDPParser): def __init__(self, *args, **kwargs): super(PMPParser, self).__init__( PMPParameter, @@ -33,7 +33,7 @@ def __init__(self, *args, **kwargs): self.use("diags") -class PMPMetricsParser(cdp.cdp_parser.CDPParser): +class PMPMetricsParser(cdp.CDPParser): def __init__(self, *args, **kwargs): super(PMPMetricsParser, self).__init__( PMPMetricsParameter, diff --git a/pcmdi_metrics/variability_mode/variability_modes_driver.py b/pcmdi_metrics/variability_mode/variability_modes_driver.py index 92b4d36b6..ab130268d 100755 --- a/pcmdi_metrics/variability_mode/variability_modes_driver.py +++ b/pcmdi_metrics/variability_mode/variability_modes_driver.py @@ -37,7 +37,7 @@ from shutil import copyfile from pcmdi_metrics.io import fill_template, get_grid, load_regions_specs, region_subset -from pcmdi_metrics.mean_climate.lib import pmp_parser +from pcmdi_metrics.utils import pmp_parser from pcmdi_metrics.stats import calculate_temporal_correlation as calcTCOR from pcmdi_metrics.stats import mean_xy from pcmdi_metrics.utils import regrid, sort_human, tree From 4e94fcfef5ee5e3bf35a4bdc9d388163ad8abf8b Mon Sep 17 00:00:00 2001 From: Kristin Chang Date: Wed, 29 Oct 2025 15:13:51 -0700 Subject: [PATCH 2/5] pre-commit fixes --- .../scripts/compositeDiurnalStatistics.py | 2 +- .../diurnal/scripts/computeStdOfDailyMeans.py | 2 +- .../diurnal/scripts/std_of_dailymeans.py | 2 +- .../diurnal/scripts/std_of_hourlyvalues.py | 3 +- .../scripts/std_of_meandiurnalcycle.py | 3 +- pcmdi_metrics/io/__init__.py | 2 +- pcmdi_metrics/io/base.py | 2 +- pcmdi_metrics/io/cdp_io.py | 3 +- pcmdi_metrics/mjo/mjo_metrics_driver.py | 3 +- .../monsoon_sperber/driver_monsoon_sperber.py | 3 +- .../precip_distribution_driver.py | 2 +- ...variability_across_timescales_PS_driver.py | 2 +- pcmdi_metrics/utils/__init__.py | 8 +- pcmdi_metrics/utils/cdp_parameter.py | 32 +- pcmdi_metrics/utils/cdp_parser.py | 305 ++++++++++++------ pcmdi_metrics/utils/cdp_run.py | 23 +- pcmdi_metrics/utils/pmp_parameter.py | 1 - pcmdi_metrics/utils/pmp_parser.py | 6 +- .../variability_modes_driver.py | 3 +- 19 files changed, 249 insertions(+), 158 deletions(-) diff --git a/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py b/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py index 152591048..3cb295251 100755 --- a/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py +++ b/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py @@ -24,7 +24,6 @@ import multiprocessing as mp import os -from pcmdi_metrics.utils import cdp_run import cftime import numpy as np import xarray as xr @@ -44,6 +43,7 @@ get_time_key, xcdat_open, ) +from pcmdi_metrics.utils import cdp_run def main(): diff --git a/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py b/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py index 6e4934f4a..2bb877d88 100755 --- a/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py +++ b/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py @@ -16,7 +16,6 @@ import multiprocessing as mp import os -from pcmdi_metrics.utils import cdp_run import cftime import xarray as xr @@ -32,6 +31,7 @@ get_longitude_key, xcdat_open, ) +from pcmdi_metrics.utils import cdp_run def main(): diff --git a/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py b/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py index ea31e31f8..9a469e530 100755 --- a/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py +++ b/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py @@ -24,7 +24,6 @@ import multiprocessing as mp import os -from pcmdi_metrics.utils import cdp_run import numpy as np import pcmdi_metrics @@ -41,6 +40,7 @@ get_time_key, xcdat_open, ) +from pcmdi_metrics.utils import cdp_run def main(): diff --git a/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py b/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py index 945c6618b..53cef3a60 100755 --- a/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py +++ b/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py @@ -21,8 +21,6 @@ import multiprocessing as mp import os -from pcmdi_metrics.utils import cdp_run - import pcmdi_metrics from pcmdi_metrics import resources from pcmdi_metrics.diurnal import compute_area_weighted_rms @@ -33,6 +31,7 @@ populateStringConstructor, ) from pcmdi_metrics.io import get_latitude_key, get_longitude_key, xcdat_open +from pcmdi_metrics.utils import cdp_run def main(): diff --git a/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py b/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py index a626b5af2..2a853c387 100755 --- a/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py +++ b/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py @@ -19,8 +19,6 @@ import multiprocessing as mp import os -from pcmdi_metrics.utils import cdp_run - import pcmdi_metrics from pcmdi_metrics import resources from pcmdi_metrics.diurnal import compute_area_weighted_rms @@ -36,6 +34,7 @@ get_time_key, xcdat_open, ) +from pcmdi_metrics.utils import cdp_run def main(): diff --git a/pcmdi_metrics/io/__init__.py b/pcmdi_metrics/io/__init__.py index 694d6d9d8..3fc20974a 100644 --- a/pcmdi_metrics/io/__init__.py +++ b/pcmdi_metrics/io/__init__.py @@ -2,7 +2,7 @@ from .xcdat_openxml import xcdat_open # noqa # isort:skip from .string_constructor import StringConstructor, fill_template # noqa # isort:skip from . import base # noqa -from . import cdp_io # noqa +from . import cdp_io # noqa from .xcdat_dataset_io import ( # noqa # isort:skip da_to_ds, diff --git a/pcmdi_metrics/io/base.py b/pcmdi_metrics/io/base.py index 9d133130f..63d9fdd2a 100755 --- a/pcmdi_metrics/io/base.py +++ b/pcmdi_metrics/io/base.py @@ -16,7 +16,7 @@ import pcmdi_metrics from pcmdi_metrics import LOG_LEVEL -from pcmdi_metrics.io import cdp_io, StringConstructor +from pcmdi_metrics.io import StringConstructor, cdp_io logging.getLogger("pcmdi_metrics").setLevel(LOG_LEVEL) # set up to log errors diff --git a/pcmdi_metrics/io/cdp_io.py b/pcmdi_metrics/io/cdp_io.py index 1bd17bcb7..0c1da22e1 100644 --- a/pcmdi_metrics/io/cdp_io.py +++ b/pcmdi_metrics/io/cdp_io.py @@ -1,6 +1,7 @@ from __future__ import print_function import abc + from six import with_metaclass @@ -13,4 +14,4 @@ def read(self): @abc.abstractmethod def write(self): """Write a file.""" - raise NotImplementedError() \ No newline at end of file + raise NotImplementedError() diff --git a/pcmdi_metrics/mjo/mjo_metrics_driver.py b/pcmdi_metrics/mjo/mjo_metrics_driver.py index ae4f03072..c1d17adc4 100755 --- a/pcmdi_metrics/mjo/mjo_metrics_driver.py +++ b/pcmdi_metrics/mjo/mjo_metrics_driver.py @@ -42,14 +42,13 @@ from shutil import copyfile import pcmdi_metrics -from pcmdi_metrics.utils import pmp_parser from pcmdi_metrics.mjo.lib import ( AddParserArgument, YearCheck, mjo_metric_ewr_calculation, mjo_metrics_to_json, ) -from pcmdi_metrics.utils import fill_template, tree +from pcmdi_metrics.utils import fill_template, pmp_parser, tree # Must be done before any CDAT library is called. # https://github.com/CDAT/cdat/issues/2213 diff --git a/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py b/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py index 6061c443f..baa758c49 100644 --- a/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py +++ b/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py @@ -54,7 +54,6 @@ from pcmdi_metrics.io import load_regions_specs, region_subset, xcdat_open from pcmdi_metrics.io.base import Base -from pcmdi_metrics.utils import pmp_parser from pcmdi_metrics.monsoon_sperber.lib import ( AddParserArgument, YearCheck, @@ -64,7 +63,7 @@ sperber_metrics, tree, ) -from pcmdi_metrics.utils import create_land_sea_mask, fill_template +from pcmdi_metrics.utils import create_land_sea_mask, fill_template, pmp_parser # How many elements each list should have n = 5 # pentad diff --git a/pcmdi_metrics/precip_distribution/precip_distribution_driver.py b/pcmdi_metrics/precip_distribution/precip_distribution_driver.py index c911ad725..a01157ddc 100644 --- a/pcmdi_metrics/precip_distribution/precip_distribution_driver.py +++ b/pcmdi_metrics/precip_distribution/precip_distribution_driver.py @@ -7,13 +7,13 @@ import xarray as xr from pcmdi_metrics.io import StringConstructor, get_calendar, xcdat_open -from pcmdi_metrics.utils.pmp_parser import PMPParser from pcmdi_metrics.precip_distribution.lib import ( AddParserArgument, Regrid_xr, precip_distribution_cum, precip_distribution_frq_amt, ) +from pcmdi_metrics.utils.pmp_parser import PMPParser # Read parameters P = PMPParser() diff --git a/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py b/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py index a4abe1657..0565122ca 100644 --- a/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py +++ b/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py @@ -2,11 +2,11 @@ import glob import os -from pcmdi_metrics.utils.pmp_parser import PMPParser from pcmdi_metrics.precip_variability.lib import ( AddParserArgument, precip_variability_across_timescale, ) +from pcmdi_metrics.utils.pmp_parser import PMPParser # Read parameters P = PMPParser() diff --git a/pcmdi_metrics/utils/__init__.py b/pcmdi_metrics/utils/__init__.py index 04128c967..32c237c01 100644 --- a/pcmdi_metrics/utils/__init__.py +++ b/pcmdi_metrics/utils/__init__.py @@ -1,4 +1,6 @@ from .adjust_units import adjust_units, fix_tuple +from .cdp_parameter import CDPParameter +from .cdp_parser import CDPParser from .custom_season import ( custom_season_average, custom_season_departure, @@ -21,6 +23,8 @@ regrid, ) from .land_sea_mask import apply_landmask, apply_oceanmask, create_land_sea_mask +from .pmp_parameter import PMPMetricsParameter, PMPParameter +from .pmp_parser import PMPMetricsParser, PMPParser from .qc import ( check_daily_time_axis, check_monthly_time_axis, @@ -31,7 +35,3 @@ from .string_constructor import StringConstructor, fill_template from .tree_dict import tree from .xr_to_cdms2 import cdms2_to_xarray, xarray_to_cdms2 -from .cdp_parser import CDPParser -from .cdp_parameter import CDPParameter -from .pmp_parser import PMPParser, PMPMetricsParser -from .pmp_parameter import PMPParameter, PMPMetricsParameter diff --git a/pcmdi_metrics/utils/cdp_parameter.py b/pcmdi_metrics/utils/cdp_parameter.py index 7b91ab6a5..8a0ba3e1f 100644 --- a/pcmdi_metrics/utils/cdp_parameter.py +++ b/pcmdi_metrics/utils/cdp_parameter.py @@ -1,12 +1,10 @@ from __future__ import print_function -import abc +import copy import importlib -import sys import os -import copy +import sys import types -from six import with_metaclass class CDPParameter(object): @@ -16,16 +14,17 @@ def __add__(self, other): """ # First make a deepcopy of the current object. duplicate = copy.deepcopy(self) - + for attr in dir(other): # Ignore any of the hidden attributes. - if attr.startswith('_') or \ - isinstance(getattr(other, attr), types.MethodType): + if attr.startswith("_") or isinstance( + getattr(other, attr), types.MethodType + ): continue val = copy.deepcopy(getattr(other, attr)) setattr(duplicate, attr, val) - + return duplicate def check_values(self): @@ -39,20 +38,21 @@ def load_parameter_from_py(self, parameter_file_path): """ Initialize a parameter object from a Python script. """ - parameter_as_module = \ - self.import_user_parameter_file_as_module(parameter_file_path) + parameter_as_module = self.import_user_parameter_file_as_module( + parameter_file_path + ) self.load_parameters_from_module(parameter_as_module) def import_user_parameter_file_as_module(self, parameter_file_path): if not os.path.isfile(parameter_file_path): - raise IOError('Parameter file %s not found.' % parameter_file_path) + raise IOError("Parameter file %s not found." % parameter_file_path) path_to_module = os.path.split(parameter_file_path)[0] module_name = os.path.split(parameter_file_path)[1] - if module_name.count('.') > 1: + if module_name.count(".") > 1: raise ValueError("Filename cannot contain '.' outside extension.") - if '.' in module_name: - module_name = module_name.split('.')[0] + if "." in module_name: + module_name = module_name.split(".")[0] sys.path.insert(0, path_to_module) return importlib.import_module(module_name) @@ -60,10 +60,10 @@ def import_user_parameter_file_as_module(self, parameter_file_path): def load_parameters_from_module(self, parameter_as_module): user_defined_parameters = [] for user_parameter in dir(parameter_as_module): - if not user_parameter.startswith('__'): + if not user_parameter.startswith("__"): user_defined_parameters.append(user_parameter) # Initialize the variables in this parameter, so the driver can # access them as if they were defined regularly. for p in user_defined_parameters: - self.__dict__[p] = getattr(parameter_as_module, p) \ No newline at end of file + self.__dict__[p] = getattr(parameter_as_module, p) diff --git a/pcmdi_metrics/utils/cdp_parser.py b/pcmdi_metrics/utils/cdp_parser.py index d2c5b1aa4..ed0f3125a 100644 --- a/pcmdi_metrics/utils/cdp_parser.py +++ b/pcmdi_metrics/utils/cdp_parser.py @@ -1,16 +1,17 @@ from __future__ import print_function -import sys import argparse -import json -import yaml -import warnings -import itertools import collections import copy -import random import hashlib +import itertools +import json +import random +import sys import types +import warnings + +import yaml if sys.version_info[0] >= 3: import configparser @@ -21,19 +22,26 @@ class CDPParser(argparse.ArgumentParser): - def __init__(self, parameter_cls=None, default_args_file=[], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, *args, **kwargs): + def __init__( + self, + parameter_cls=None, + default_args_file=[], + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + *args, + **kwargs, + ): # conflict_handler='resolve' lets new args override older ones self.__default_args = [] - super(CDPParser, self).__init__(conflict_handler='resolve', - formatter_class=formatter_class, - *args, **kwargs) + super(CDPParser, self).__init__( + conflict_handler="resolve", formatter_class=formatter_class, *args, **kwargs + ) self.load_default_args(default_args_file) self.__args_namespace = None self.__parameter_cls = parameter_cls - + if not self.__parameter_cls: from cdp.cdp_parameter import CDPParameter + self.__parameter_cls = CDPParameter def parse_args(self, args=None, namespace=None): @@ -46,7 +54,7 @@ def parse_args(self, args=None, namespace=None): return super(CDPParser, self).parse_args(args, namespace) def view_args(self): - """" + """ " Returns the args namespace. """ self._parse_arguments() @@ -60,7 +68,7 @@ def _was_command_used(self, cmdline_arg): # self.cmd_used is like: ['something.py', '-p', 'test.py', '--s1', 'something'] for cmd in self.cmd_used: # Sometimes, a command is run with '=': 'driver.py --something=this' - for c in cmd.split('='): + for c in cmd.split("="): if cmdline_arg == c: return True return False @@ -72,7 +80,9 @@ def _is_arg_default_value(self, arg): """ # Each cmdline_arg is either '-*' or '--*'. for cmdline_arg in self._option_string_actions: - if arg == self._option_string_actions[cmdline_arg].dest and self._was_command_used(cmdline_arg): + if arg == self._option_string_actions[ + cmdline_arg + ].dest and self._was_command_used(cmdline_arg): return False return True @@ -91,7 +101,9 @@ def _get_default_from_cmdline(self, parameters): but only if that parameter is NOT already defined. """ for arg_name, arg_value in vars(self.__args_namespace).items(): - if self._is_arg_default_value(arg_name) and not hasattr(parameters, arg_name): + if self._is_arg_default_value(arg_name) and not hasattr( + parameters, arg_name + ): setattr(parameters, arg_name, arg_value) def _parse_arguments(self): @@ -99,14 +111,14 @@ def _parse_arguments(self): Parse the command line arguments while checking for the user's arguments. """ if self.__args_namespace is None: - self.__args_namespace = self.parse_args() + self.__args_namespace = self.parse_args() def get_orig_parameters(self, check_values=False, argparse_vals_only=True): """ Returns the parameters created by -p. If -p wasn't used, returns None. """ self._parse_arguments() - + if not self.__args_namespace.parameters: return None @@ -116,8 +128,7 @@ def get_orig_parameters(self, check_values=False, argparse_vals_only=True): parameter.__dict__.clear() # if self.__args_namespace.parameters is not None: - parameter.load_parameter_from_py( - self.__args_namespace.parameters) + parameter.load_parameter_from_py(self.__args_namespace.parameters) if check_values: parameter.check_values() @@ -126,7 +137,9 @@ def get_orig_parameters(self, check_values=False, argparse_vals_only=True): return parameter - def get_parameters_from_json(self, json_file, check_values=False, argparse_vals_only=True): + def get_parameters_from_json( + self, json_file, check_values=False, argparse_vals_only=True + ): """ Given a json file, return the parameters from it. """ @@ -164,31 +177,37 @@ def _create_cfg_hash_titles(self, cfg_file): h_sha256 = hashlib.sha256() i = 0 - + while i < len(lines): - if lines[i] in ['[#]\n', '[#]']: + if lines[i] in ["[#]\n", "[#]"]: replace_idx = i str_list = [] i += 1 - while i < len(lines) and not lines[i].startswith('['): + while i < len(lines) and not lines[i].startswith("["): str_list.append(lines[i]) i += 1 str_list.append(str(random.random())) # Randomize the hash even more. - h_sha256.update(''.join(str_list).encode()) - lines[replace_idx] = '[{}]'.format(h_sha256.hexdigest()) + h_sha256.update("".join(str_list).encode()) + lines[replace_idx] = "[{}]".format(h_sha256.hexdigest()) else: i += 1 - return StringIO('\n'.join(lines)) + return StringIO("\n".join(lines)) - def get_parameters_from_cfg(self, cfg_file, check_values=False, argparse_vals_only=True): + def get_parameters_from_cfg( + self, cfg_file, check_values=False, argparse_vals_only=True + ): """ Given a cfg file, return the parameters from it. """ parameters = [] cfg_file_obj = self._create_cfg_hash_titles(cfg_file) - kwargs = {'strict': False} if sys.version_info[0] >= 3 else {} # 'strict' keyword doesn't work in Python 2. - config = configparser.ConfigParser(**kwargs) # Allow for two lines to be the same. + kwargs = ( + {"strict": False} if sys.version_info[0] >= 3 else {} + ) # 'strict' keyword doesn't work in Python 2. + config = configparser.ConfigParser( + **kwargs + ) # Allow for two lines to be the same. config.readfp(cfg_file_obj) for section in config.sections(): @@ -210,13 +229,15 @@ def get_parameters_from_cfg(self, cfg_file, check_values=False, argparse_vals_on return parameters - def get_other_parameters(self, files_to_open=[], check_values=False, argparse_vals_only=True): + def get_other_parameters( + self, files_to_open=[], check_values=False, argparse_vals_only=True + ): """ - Returns the parameters created by -d. If files_to_open is defined, + Returns the parameters created by -d. If files_to_open is defined, then use the path specified instead of -d. """ parameters = [] - + self._parse_arguments() if files_to_open == []: @@ -224,13 +245,18 @@ def get_other_parameters(self, files_to_open=[], check_values=False, argparse_va if files_to_open is not None: for diags_file in files_to_open: - if '.json' in diags_file: - params = self.get_parameters_from_json(diags_file, check_values, argparse_vals_only) - elif '.cfg' in diags_file: - params = self.get_parameters_from_cfg(diags_file, check_values, argparse_vals_only) + if ".json" in diags_file: + params = self.get_parameters_from_json( + diags_file, check_values, argparse_vals_only + ) + elif ".cfg" in diags_file: + params = self.get_parameters_from_cfg( + diags_file, check_values, argparse_vals_only + ) else: raise RuntimeError( - 'The parameters input file must be either a .json or .cfg file') + "The parameters input file must be either a .json or .cfg file" + ) for p in params: parameters.append(p) @@ -242,7 +268,12 @@ def _were_cmdline_args_used(self): Checks that other parameters, besides '-p' or '-d', were used. """ for cmd in self.cmd_used: - if cmd.startswith('-') and cmd not in ['-p', '--parameters', '-d', '--diags']: + if cmd.startswith("-") and cmd not in [ + "-p", + "--parameters", + "-d", + "--diags", + ]: return True return False @@ -261,7 +292,7 @@ def get_cmdline_parameters(self, check_values=False, argparse_vals_only=True): self._parse_arguments() if not self._were_cmdline_args_used(): - return None + return None parameter = self.__parameter_cls() @@ -299,40 +330,61 @@ def add_default_values(self, parameter, default_vars=False, cmd_default_vars=Fal # Add the command line default parameters first. if cmd_default_vars: for arg_name, arg_value in vars(self.__args_namespace).items(): - if arg_name in parameter.__dict__ or not self._is_arg_default_value(arg_name): + if arg_name in parameter.__dict__ or not self._is_arg_default_value( + arg_name + ): continue # Only add the default values, that aren't already in parameter. setattr(parameter, arg_name, arg_value) - + # Then add the defaults defined in the Parameter class. if default_vars: - for arg_name, arg_value in vars(self.__parameter_cls()).items(): + for arg_name, arg_value in vars(self.__parameter_cls()).items(): if arg_name in parameter.__dict__: continue setattr(parameter, arg_name, arg_value) - def _get_selectors(self, cmdline_parameters=None, orig_parameters=None, other_parameters=None): + def _get_selectors( + self, cmdline_parameters=None, orig_parameters=None, other_parameters=None + ): """ Look through the cmdline_parameters, orig_parameters, and other_parameters in that order for the selectors used. If not defined in any of them, use the default one in the class. """ - if hasattr(cmdline_parameters, 'selectors') and cmdline_parameters.selectors is not None: + if ( + hasattr(cmdline_parameters, "selectors") + and cmdline_parameters.selectors is not None + ): return cmdline_parameters.selectors - elif hasattr(orig_parameters, 'selectors') and orig_parameters.selectors is not None: + elif ( + hasattr(orig_parameters, "selectors") + and orig_parameters.selectors is not None + ): return orig_parameters.selectors - elif hasattr(other_parameters, 'selectors') and other_parameters.selectors is not None: + elif ( + hasattr(other_parameters, "selectors") + and other_parameters.selectors is not None + ): return other_parameters.selectors else: # If the parameter class has selectors, try to add that in. param = self.__parameter_cls() - if hasattr(param, 'selectors'): + if hasattr(param, "selectors"): return param.selectors # None of the passed in parameters have a selector and neither the main_parameter # nor the parameter class has selectors, so return an empty list. return [] - def combine_params(self, cmdline_parameters=None, orig_parameters=None, other_parameters=None, vars_to_ignore=[], default_vars=False, cmd_default_vars=False): + def combine_params( + self, + cmdline_parameters=None, + orig_parameters=None, + other_parameters=None, + vars_to_ignore=[], + default_vars=False, + cmd_default_vars=False, + ): """ Combine cmdline_params (-* or --*), orig_parameters (-p), and other_parameters (-d), while ignoring any parameters listed in the 'selectors' parameter. @@ -368,13 +420,15 @@ def combine_params(self, cmdline_parameters=None, orig_parameters=None, other_pa elif orig_parameters: self.add_default_values(orig_parameters, default_vars, cmd_default_vars) elif cmdline_parameters: - self.add_default_values(cmdline_parameters, default_vars, cmd_default_vars) + self.add_default_values( + cmdline_parameters, default_vars, cmd_default_vars + ) def combine_orig_and_other_params(self, orig_parameters, other_parameters): """ Combine orig_parameters with all of the other_parameters. """ - print('Depreciation warning: please use combine_params() instead') + print("Depreciation warning: please use combine_params() instead") self.combine_params(None, orig_parameters, other_parameters) def granulate(self, parameters): @@ -384,7 +438,9 @@ def granulate(self, parameters): """ final_parameters = [] for param in parameters: - if not hasattr(param, 'granulate') or (hasattr(param, 'granulate') and not param.granulate): + if not hasattr(param, "granulate") or ( + hasattr(param, "granulate") and not param.granulate + ): final_parameters.append(param) continue @@ -405,10 +461,16 @@ def granulate(self, parameters): vals_to_granulate = collections.OrderedDict() for v in vars_to_granulate: if not hasattr(param, v): - raise RuntimeError("Parameters object has no attribute '{}' to granulate.".format(v)) + raise RuntimeError( + "Parameters object has no attribute '{}' to granulate.".format( + v + ) + ) param_v = getattr(param, v) if not isinstance(param_v, collections.Iterable): - raise RuntimeError("Granulate option '{}' is not an iterable.".format(v)) + raise RuntimeError( + "Granulate option '{}' is not an iterable.".format(v) + ) if param_v: # Ignore []. vals_to_granulate[v] = param_v @@ -428,8 +490,9 @@ def granulate(self, parameters): def select(self, main_parameters, parameters): """ Given a list of parameters (parameters), only return those from this list - whose 'selector' parameters are a subset of the 'selector' parameters of main_parameters. + whose 'selector' parameters are a subset of the 'selector' parameters of main_parameters. """ + def is_subset(param1, param2): """ Check if param1 is a subset of param2. @@ -445,35 +508,38 @@ def is_subset(param1, param2): # Can't select from None. if not main_parameters: return parameters - + selectors = self._get_selectors(None, main_parameters, parameters) final_parameters = [] for param in parameters: - if all(is_subset(getattr(param, select_parameter), - getattr(main_parameters, select_parameter)) - for select_parameter in selectors): - final_parameters.append(param) + if all( + is_subset( + getattr(param, select_parameter), + getattr(main_parameters, select_parameter), + ) + for select_parameter in selectors + ): + final_parameters.append(param) return final_parameters - def _get_alias(self, param): """ For a single parameter, get the aliases of it. """ # Parameters can start with either '-' or '--'. - param = '--{}'.format(param) + param = "--{}".format(param) if param not in self._option_string_actions: - param = '-{}'.format(param) + param = "-{}".format(param) if param not in self._option_string_actions: return [] # Ex: If param is 'parameters', then we get ['-p', '--parameters']. aliases = self._option_string_actions[param].option_strings - return [a.replace('-', '') for a in aliases] + return [a.replace("-", "") for a in aliases] def add_aliases(self, parameters): """ @@ -495,7 +561,16 @@ def add_aliases(self, parameters): for alias in aliases: setattr(param, alias, param_value) - def get_parameters(self, cmdline_parameters=None, orig_parameters=None, other_parameters=[], default_vars=True, cmd_default_vars=True, *args, **kwargs): + def get_parameters( + self, + cmdline_parameters=None, + orig_parameters=None, + other_parameters=[], + default_vars=True, + cmd_default_vars=True, + *args, + **kwargs, + ): """ Get the parameters based on the command line arguments and return a list of them. """ @@ -505,11 +580,20 @@ def get_parameters(self, cmdline_parameters=None, orig_parameters=None, other_pa orig_parameters = self.get_orig_parameters(*args, **kwargs) if other_parameters == []: other_parameters = self.get_other_parameters(*args, **kwargs) - + # We don't want to add the selectors to each of the parameters. # Because if we do, it'll select all of the parameters at the end during the selection step. - vars_to_ignore = self._get_selectors(cmdline_parameters, orig_parameters, other_parameters) - self.combine_params(cmdline_parameters, orig_parameters, other_parameters, vars_to_ignore, default_vars, cmd_default_vars) + vars_to_ignore = self._get_selectors( + cmdline_parameters, orig_parameters, other_parameters + ) + self.combine_params( + cmdline_parameters, + orig_parameters, + other_parameters, + vars_to_ignore, + default_vars, + cmd_default_vars, + ) if other_parameters != []: final_parameters = other_parameters @@ -552,7 +636,8 @@ def get_parameter(self, warning=False, *args, **kwargs): """ if warning: print( - 'Depreciation warning: Use get_parameters() instead, which returns a list of Parameters.') + "Depreciation warning: Use get_parameters() instead, which returns a list of Parameters." + ) return self.get_parameters(*args, **kwargs)[0] def load_default_args_from_json(self, files): @@ -561,7 +646,6 @@ def load_default_args_from_json(self, files): """ # This is needed for the loading from JSON files, # because the type can be ast.literal_eval. - import ast if not isinstance(files, (list, tuple)): files = [files] @@ -583,18 +667,19 @@ def load_default_args_from_json(self, files): # hence not setting it. try: params["type"] = eval(params.pop("type", "str")) - except: + except Exception: pass self.store_default_arguments(option_strings, params) success = True - except: - warnings.warn("Failed to load param {} from json file {}".format( - k, afile)) + except Exception: + warnings.warn( + "Failed to load param {} from json file {}".format(k, afile) + ) return success def store_default_arguments(self, options, params): - self.__default_args.insert(0,([options, params])) + self.__default_args.insert(0, ([options, params])) def print_available_defaults(self): p = argparse.ArgumentParser() @@ -625,7 +710,9 @@ def use(self, options): else: raise RuntimeError( "Could not match {} to any of the default arguments {}".format( - option, self.available_defaults)) + option, self.available_defaults + ) + ) def load_default_args(self, files=[]): """ @@ -634,49 +721,59 @@ def load_default_args(self, files=[]): if self.load_default_args_from_json(files): return self.add_argument( - '-p', '--parameters', + "-p", + "--parameters", type=str, - dest='parameters', - help='Path to the user-defined parameter file.', - required=False) + dest="parameters", + help="Path to the user-defined parameter file.", + required=False, + ) self.add_argument( - '-d', '--diags', + "-d", + "--diags", type=str, - nargs='+', - dest='other_parameters', + nargs="+", + dest="other_parameters", default=[], - help='Path to the other user-defined parameter file.', - required=False) + help="Path to the other user-defined parameter file.", + required=False, + ) self.add_argument( - '-n', '--num_workers', + "-n", + "--num_workers", type=int, - dest='num_workers', - help='Number of workers, used when running with multiprocessing or in distributed mode.', - required=False) + dest="num_workers", + help="Number of workers, used when running with multiprocessing or in distributed mode.", + required=False, + ) self.add_argument( - '--scheduler_addr', + "--scheduler_addr", type=str, - dest='scheduler_addr', - help='Address of the scheduler in the form of IP_ADDRESS:PORT. Used when running in distributed mode.', - required=False) + dest="scheduler_addr", + help="Address of the scheduler in the form of IP_ADDRESS:PORT. Used when running in distributed mode.", + required=False, + ) self.add_argument( - '-g', '--granulate', + "-g", + "--granulate", type=str, - nargs='+', - dest='granulate', - help='A list of variables to granulate.', - required=False) + nargs="+", + dest="granulate", + help="A list of variables to granulate.", + required=False, + ) self.add_argument( - '--selectors', + "--selectors", type=str, - nargs='+', - dest='selectors', - help='A list of variables to be used to select parameters from.', - required=False) + nargs="+", + dest="selectors", + help="A list of variables to be used to select parameters from.", + required=False, + ) def add_args_and_values(self, arg_list): """ Used for testing. Can test args input as if they were inputted from the command line. """ - self.__args_namespace = self.parse_args(arg_list) \ No newline at end of file + self.__args_namespace = self.parse_args(arg_list) diff --git a/pcmdi_metrics/utils/cdp_run.py b/pcmdi_metrics/utils/cdp_run.py index 7e13b6de6..862454c84 100644 --- a/pcmdi_metrics/utils/cdp_run.py +++ b/pcmdi_metrics/utils/cdp_run.py @@ -13,6 +13,7 @@ def serial(func, parameters): results.append(func(p)) return results + def multiprocess(func, parameters, num_workers=None, context=None): """ Run the function with the parameters in parallel using multiprocessing. @@ -23,17 +24,16 @@ def multiprocess(func, parameters, num_workers=None, context=None): """ bag = dask.bag.from_sequence(parameters) - config = {'scheduler': 'processes'} + config = {"scheduler": "processes"} if context is not None: - config['multiprocessing.context'] = context - elif hasattr(parameters[0], 'multiprocessing_context'): - config['multiprocessing.context'] = \ - parameters[0].multiprocessing_context + config["multiprocessing.context"] = context + elif hasattr(parameters[0], "multiprocessing_context"): + config["multiprocessing.context"] = parameters[0].multiprocessing_context with dask.config.set(config): if num_workers: results = bag.map(func).compute(num_workers=num_workers) - elif hasattr(parameters[0], 'num_workers'): + elif hasattr(parameters[0], "num_workers"): results = bag.map(func).compute(num_workers=parameters[0].num_workers) else: # num of workers is defaulted to the number of logical processes @@ -41,6 +41,7 @@ def multiprocess(func, parameters, num_workers=None, context=None): return results + def distribute(func, parameters, scheduler_addr=None): """ Run the function with the parameters in parallel distributedly. @@ -48,8 +49,10 @@ def distribute(func, parameters, scheduler_addr=None): try: if scheduler_addr: addr = scheduler_addr - elif not hasattr(parameters[0], 'scheduler_addr'): - raise RuntimeError('The parameters or distribute() need a scheduler_addr parameter.') + elif not hasattr(parameters[0], "scheduler_addr"): + raise RuntimeError( + "The parameters or distribute() need a scheduler_addr parameter." + ) else: addr = parameters[0].scheduler_addr @@ -57,9 +60,9 @@ def distribute(func, parameters, scheduler_addr=None): results = client.map(func, parameters) client.gather(results) except Exception as e: - print('Distributed run failed.') + print("Distributed run failed.") raise e finally: client.close() - return results \ No newline at end of file + return results diff --git a/pcmdi_metrics/utils/pmp_parameter.py b/pcmdi_metrics/utils/pmp_parameter.py index 1d69d7183..62259bd99 100755 --- a/pcmdi_metrics/utils/pmp_parameter.py +++ b/pcmdi_metrics/utils/pmp_parameter.py @@ -2,7 +2,6 @@ import os import pcmdi_metrics.utils as cdp - from pcmdi_metrics import LOG_LEVEL from pcmdi_metrics.utils import StringConstructor diff --git a/pcmdi_metrics/utils/pmp_parser.py b/pcmdi_metrics/utils/pmp_parser.py index 0cab41d4a..21f6f8cbb 100644 --- a/pcmdi_metrics/utils/pmp_parser.py +++ b/pcmdi_metrics/utils/pmp_parser.py @@ -1,12 +1,8 @@ import os import pcmdi_metrics.utils.cdp_parser as cdp - from pcmdi_metrics import resources -from pcmdi_metrics.utils.pmp_parameter import ( - PMPMetricsParameter, - PMPParameter, -) +from pcmdi_metrics.utils.pmp_parameter import PMPMetricsParameter, PMPParameter try: basestring # noqa diff --git a/pcmdi_metrics/variability_mode/variability_modes_driver.py b/pcmdi_metrics/variability_mode/variability_modes_driver.py index ab130268d..1e48aaf2f 100755 --- a/pcmdi_metrics/variability_mode/variability_modes_driver.py +++ b/pcmdi_metrics/variability_mode/variability_modes_driver.py @@ -37,10 +37,9 @@ from shutil import copyfile from pcmdi_metrics.io import fill_template, get_grid, load_regions_specs, region_subset -from pcmdi_metrics.utils import pmp_parser from pcmdi_metrics.stats import calculate_temporal_correlation as calcTCOR from pcmdi_metrics.stats import mean_xy -from pcmdi_metrics.utils import regrid, sort_human, tree +from pcmdi_metrics.utils import pmp_parser, regrid, sort_human, tree from pcmdi_metrics.variability_mode.lib import ( AddParserArgument, VariabilityModeCheck, From 6fc499d6364a352f12b9e1a4eb40bddd5f2a3dba Mon Sep 17 00:00:00 2001 From: Kristin Chang Date: Wed, 29 Oct 2025 16:14:55 -0700 Subject: [PATCH 3/5] Update pmp_parameter.py --- pcmdi_metrics/utils/pmp_parameter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pcmdi_metrics/utils/pmp_parameter.py b/pcmdi_metrics/utils/pmp_parameter.py index 62259bd99..7951b5aac 100755 --- a/pcmdi_metrics/utils/pmp_parameter.py +++ b/pcmdi_metrics/utils/pmp_parameter.py @@ -3,7 +3,8 @@ import pcmdi_metrics.utils as cdp from pcmdi_metrics import LOG_LEVEL -from pcmdi_metrics.utils import StringConstructor + +from . import StringConstructor try: basestring # noqa From 681c6775be82559fa11d7d003eb8c7a1dab70733 Mon Sep 17 00:00:00 2001 From: Kristin Chang Date: Wed, 29 Oct 2025 16:19:27 -0700 Subject: [PATCH 4/5] Update pmp_parameter.py --- pcmdi_metrics/utils/pmp_parameter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcmdi_metrics/utils/pmp_parameter.py b/pcmdi_metrics/utils/pmp_parameter.py index 7951b5aac..00109753f 100755 --- a/pcmdi_metrics/utils/pmp_parameter.py +++ b/pcmdi_metrics/utils/pmp_parameter.py @@ -4,7 +4,7 @@ import pcmdi_metrics.utils as cdp from pcmdi_metrics import LOG_LEVEL -from . import StringConstructor +from .pmp_parameter import StringConstructor try: basestring # noqa From 6892a46162845b3dace40da962b4616ac55aa008 Mon Sep 17 00:00:00 2001 From: Kristin Chang Date: Wed, 29 Oct 2025 16:23:56 -0700 Subject: [PATCH 5/5] Update pmp_parameter.py --- pcmdi_metrics/utils/pmp_parameter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcmdi_metrics/utils/pmp_parameter.py b/pcmdi_metrics/utils/pmp_parameter.py index 00109753f..f1ba249ba 100755 --- a/pcmdi_metrics/utils/pmp_parameter.py +++ b/pcmdi_metrics/utils/pmp_parameter.py @@ -4,7 +4,7 @@ import pcmdi_metrics.utils as cdp from pcmdi_metrics import LOG_LEVEL -from .pmp_parameter import StringConstructor +from .string_constructor import StringConstructor try: basestring # noqa