Merge pull request #1340 from PCMDI/1339_kristinchang3_refactorCDP

kristinchang3 · web-flow · commit 0fbb014b3c13 · 2025-10-29T16:36:29.000-07:00
Add CDP to PMP utils
diff --git a/conda-env/ci.yml b/conda-env/ci.yml
@@ -13,7 +13,6 @@ dependencies:
     - numpy >=2.0.0,<3
     - cartopy >=0.22.0
     - matplotlib >=3.7.1
-    - cdp >=1.7.0
     - eofs >=2.0.0
     - seaborn >=0.12.2
     #- enso_metrics >=1.1.5
diff --git a/conda-env/dev.yml b/conda-env/dev.yml
@@ -13,7 +13,6 @@ dependencies:
     - numpy >=2.0.0,<3
     - cartopy >=0.22.0
     - matplotlib >=3.7.1
-    - cdp >=1.7.0
     - eofs >=2.0.0
     - seaborn >=0.12.2
     #- enso_metrics >=1.1.5
diff --git a/pcmdi_metrics/cloud_feedback/lib/argparse_functions.py b/pcmdi_metrics/cloud_feedback/lib/argparse_functions.py
@@ -1,6 +1,6 @@
 import argparse
 
-from cdp.cdp_parser import CDPParser
+from pcmdi_metrics.utils.cdp_parser import CDPParser
 
 
 def AddParserArgument():
diff --git a/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py b/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py
@@ -24,7 +24,6 @@
 import multiprocessing as mp
 import os
 
-import cdp
 import cftime
 import numpy as np
 import xarray as xr
@@ -44,6 +43,7 @@
     get_time_key,
     xcdat_open,
 )
+from pcmdi_metrics.utils import cdp_run
 
 
 def main():
@@ -252,7 +252,7 @@ def compute(params):
     print("FILES:", fileList)
     params = [INPUT(args, name, template) for name in fileList]
     print("PARAMS:", params)
-    cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
 
 def add_one_month(t):
diff --git a/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py b/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py
@@ -16,7 +16,6 @@
 import multiprocessing as mp
 import os
 
-import cdp
 import cftime
 import xarray as xr
 
@@ -32,6 +31,7 @@
     get_longitude_key,
     xcdat_open,
 )
+from pcmdi_metrics.utils import cdp_run
 
 
 def main():
@@ -195,7 +195,7 @@ def compute(params, debug=False):
     params = [INPUT(args, name, template) for name in fileList]
     print("PARAMS:", params)
 
-    cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
 
 def add_one_month(t):
diff --git a/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py b/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py
@@ -24,7 +24,6 @@
 import multiprocessing as mp
 import os
 
-import cdp
 import numpy as np
 
 import pcmdi_metrics
@@ -41,6 +40,7 @@
     get_time_key,
     xcdat_open,
 )
+from pcmdi_metrics.utils import cdp_run
 
 
 def main():
@@ -167,7 +167,7 @@ def compute(param):
     params = [INPUT(args, name, template) for name in files]
     print("PARAMS:", params)
 
-    results = cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    results = cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
     for r in results:
         m, region, res = r
diff --git a/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py b/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py
@@ -21,8 +21,6 @@
 import multiprocessing as mp
 import os
 
-import cdp
-
 import pcmdi_metrics
 from pcmdi_metrics import resources
 from pcmdi_metrics.diurnal import compute_area_weighted_rms
@@ -33,6 +31,7 @@
     populateStringConstructor,
 )
 from pcmdi_metrics.io import get_latitude_key, get_longitude_key, xcdat_open
+from pcmdi_metrics.utils import cdp_run
 
 
 def main():
@@ -172,7 +171,7 @@ def compute(param):
     params = [INPUT(args, name, template) for name in files]
     print("PARAMS:", params)
 
-    results = cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    results = cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
     for r in results:
         m, region, res = r
diff --git a/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py b/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py
@@ -19,8 +19,6 @@
 import multiprocessing as mp
 import os
 
-import cdp
-
 import pcmdi_metrics
 from pcmdi_metrics import resources
 from pcmdi_metrics.diurnal import compute_area_weighted_rms
@@ -36,6 +34,7 @@
     get_time_key,
     xcdat_open,
 )
+from pcmdi_metrics.utils import cdp_run
 
 
 def main():
@@ -182,7 +181,7 @@ def compute(param):
     params = [INPUT(args, name, template) for name in files]
     print("PARAMS:", params)
 
-    results = cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    results = cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
     for r in results:
         m, region, res = r
diff --git a/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py b/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from pcmdi_metrics.mean_climate.lib import pmp_parser
+from pcmdi_metrics.utils import pmp_parser
 
 
 def create_extremes_parser():
diff --git a/pcmdi_metrics/enso/lib/enso_lib.py b/pcmdi_metrics/enso/lib/enso_lib.py
@@ -7,7 +7,7 @@
 from collections import defaultdict
 
 import pcmdi_metrics
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser
+from pcmdi_metrics.utils.pmp_parser import PMPParser
 
 
 def AddParserArgument():
diff --git a/pcmdi_metrics/io/__init__.py b/pcmdi_metrics/io/__init__.py
@@ -2,6 +2,7 @@
 from .xcdat_openxml import xcdat_open  # noqa  # isort:skip
 from .string_constructor import StringConstructor, fill_template  # noqa  # isort:skip
 from . import base  # noqa
+from . import cdp_io  # noqa
 
 from .xcdat_dataset_io import (  # noqa  # isort:skip
     da_to_ds,
diff --git a/pcmdi_metrics/io/base.py b/pcmdi_metrics/io/base.py
@@ -11,13 +11,12 @@
 from datetime import datetime
 from subprocess import PIPE, Popen
 
-import cdp.cdp_io
 import numpy
 import requests
 
 import pcmdi_metrics
 from pcmdi_metrics import LOG_LEVEL
-from pcmdi_metrics.io import StringConstructor
+from pcmdi_metrics.io import StringConstructor, cdp_io
 
 logging.getLogger("pcmdi_metrics").setLevel(LOG_LEVEL)  # set up to log errors
 
@@ -264,7 +263,7 @@ def alphanum(key):
 # ----------
 
 
-class Base(cdp.cdp_io.CDPIO, StringConstructor):
+class Base(cdp_io.CDPIO, StringConstructor):
     def __init__(self, root, file_template, file_mask_template=None):
         StringConstructor.__init__(self, root + "/" + file_template)
         self.root = root
diff --git a/pcmdi_metrics/io/cdp_io.py b/pcmdi_metrics/io/cdp_io.py
@@ -0,0 +1,17 @@
+from __future__ import print_function
+
+import abc
+
+from six import with_metaclass
+
+
+class CDPIO(with_metaclass(abc.ABCMeta, object)):
+    @abc.abstractmethod
+    def read(self):
+        """Read a file."""
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def write(self):
+        """Write a file."""
+        raise NotImplementedError()
diff --git a/pcmdi_metrics/mean_climate/lib/create_mean_climate_parser.py b/pcmdi_metrics/mean_climate/lib/create_mean_climate_parser.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 import ast
 
-from pcmdi_metrics.mean_climate.lib import pmp_parser
+from pcmdi_metrics.utils import pmp_parser
 
 
 def create_mean_climate_parser():
diff --git a/pcmdi_metrics/mean_climate/pcmdi_compute_climatologies.py b/pcmdi_metrics/mean_climate/pcmdi_compute_climatologies.py
@@ -4,7 +4,7 @@
 
 from pcmdi_metrics.io import StringConstructor
 from pcmdi_metrics.mean_climate.lib import calculate_climatology
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPMetricsParser
+from pcmdi_metrics.utils.pmp_parser import PMPMetricsParser
 
 ver = datetime.datetime.now().strftime("v%Y%m%d")
 
diff --git a/pcmdi_metrics/mjo/mjo_metrics_driver.py b/pcmdi_metrics/mjo/mjo_metrics_driver.py
@@ -42,14 +42,13 @@
 from shutil import copyfile
 
 import pcmdi_metrics
-from pcmdi_metrics.mean_climate.lib import pmp_parser
 from pcmdi_metrics.mjo.lib import (
     AddParserArgument,
     YearCheck,
     mjo_metric_ewr_calculation,
     mjo_metrics_to_json,
 )
-from pcmdi_metrics.utils import fill_template, tree
+from pcmdi_metrics.utils import fill_template, pmp_parser, tree
 
 # Must be done before any CDAT library is called.
 # https://github.com/CDAT/cdat/issues/2213
diff --git a/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py b/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py
@@ -54,7 +54,6 @@
 
 from pcmdi_metrics.io import load_regions_specs, region_subset, xcdat_open
 from pcmdi_metrics.io.base import Base
-from pcmdi_metrics.mean_climate.lib import pmp_parser
 from pcmdi_metrics.monsoon_sperber.lib import (
     AddParserArgument,
     YearCheck,
@@ -64,7 +63,7 @@
     sperber_metrics,
     tree,
 )
-from pcmdi_metrics.utils import create_land_sea_mask, fill_template
+from pcmdi_metrics.utils import create_land_sea_mask, fill_template, pmp_parser
 
 # How many elements each list should have
 n = 5  # pentad
diff --git a/pcmdi_metrics/monsoon_wang/lib/argparse_functions.py b/pcmdi_metrics/monsoon_wang/lib/argparse_functions.py
@@ -1,4 +1,4 @@
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser
+from pcmdi_metrics.utils.pmp_parser import PMPParser
 
 
 def create_monsoon_wang_parser():
diff --git a/pcmdi_metrics/precip_distribution/precip_distribution_driver.py b/pcmdi_metrics/precip_distribution/precip_distribution_driver.py
@@ -7,13 +7,13 @@
 import xarray as xr
 
 from pcmdi_metrics.io import StringConstructor, get_calendar, xcdat_open
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser
 from pcmdi_metrics.precip_distribution.lib import (
     AddParserArgument,
     Regrid_xr,
     precip_distribution_cum,
     precip_distribution_frq_amt,
 )
+from pcmdi_metrics.utils.pmp_parser import PMPParser
 
 # Read parameters
 P = PMPParser()
diff --git a/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py b/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py
@@ -2,11 +2,11 @@
 import glob
 import os
 
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser
 from pcmdi_metrics.precip_variability.lib import (
     AddParserArgument,
     precip_variability_across_timescale,
 )
+from pcmdi_metrics.utils.pmp_parser import PMPParser
 
 # Read parameters
 P = PMPParser()
diff --git a/pcmdi_metrics/sea_ice/lib/sea_ice_parser.py b/pcmdi_metrics/sea_ice/lib/sea_ice_parser.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from pcmdi_metrics.mean_climate.lib import pmp_parser
+from pcmdi_metrics.utils import pmp_parser
 
 
 def create_sea_ice_parser():
diff --git a/pcmdi_metrics/utils/__init__.py b/pcmdi_metrics/utils/__init__.py
@@ -1,4 +1,6 @@
 from .adjust_units import adjust_units, fix_tuple
+from .cdp_parameter import CDPParameter
+from .cdp_parser import CDPParser
 from .custom_season import (
     custom_season_average,
     custom_season_departure,
@@ -21,6 +23,8 @@
     regrid,
 )
 from .land_sea_mask import apply_landmask, apply_oceanmask, create_land_sea_mask
+from .pmp_parameter import PMPMetricsParameter, PMPParameter
+from .pmp_parser import PMPMetricsParser, PMPParser
 from .qc import (
     check_daily_time_axis,
     check_monthly_time_axis,
diff --git a/pcmdi_metrics/utils/cdp_parameter.py b/pcmdi_metrics/utils/cdp_parameter.py
@@ -0,0 +1,69 @@
+from __future__ import print_function
+
+import copy
+import importlib
+import os
+import sys
+import types
+
+
+class CDPParameter(object):
+    def __add__(self, other):
+        """
+        Deepcopy any attribute of `other` into self.
+        """
+        # First make a deepcopy of the current object.
+        duplicate = copy.deepcopy(self)
+
+        for attr in dir(other):
+            # Ignore any of the hidden attributes.
+            if attr.startswith("_") or isinstance(
+                getattr(other, attr), types.MethodType
+            ):
+                continue
+
+            val = copy.deepcopy(getattr(other, attr))
+            setattr(duplicate, attr, val)
+
+        return duplicate
+
+    def check_values(self):
+        """
+        Check that all of the variables in
+        this parameter file are valid.
+        """
+        pass
+
+    def load_parameter_from_py(self, parameter_file_path):
+        """
+        Initialize a parameter object from a Python script.
+        """
+        parameter_as_module = self.import_user_parameter_file_as_module(
+            parameter_file_path
+        )
+        self.load_parameters_from_module(parameter_as_module)
+
+    def import_user_parameter_file_as_module(self, parameter_file_path):
+        if not os.path.isfile(parameter_file_path):
+            raise IOError("Parameter file %s not found." % parameter_file_path)
+
+        path_to_module = os.path.split(parameter_file_path)[0]
+        module_name = os.path.split(parameter_file_path)[1]
+        if module_name.count(".") > 1:
+            raise ValueError("Filename cannot contain '.' outside extension.")
+        if "." in module_name:
+            module_name = module_name.split(".")[0]
+
+        sys.path.insert(0, path_to_module)
+        return importlib.import_module(module_name)
+
+    def load_parameters_from_module(self, parameter_as_module):
+        user_defined_parameters = []
+        for user_parameter in dir(parameter_as_module):
+            if not user_parameter.startswith("__"):
+                user_defined_parameters.append(user_parameter)
+
+        # Initialize the variables in this parameter, so the driver can
+        # access them as if they were defined regularly.
+        for p in user_defined_parameters:
+            self.__dict__[p] = getattr(parameter_as_module, p)
diff --git a/pcmdi_metrics/utils/cdp_parser.py b/pcmdi_metrics/utils/cdp_parser.py
diff --git a/pcmdi_metrics/utils/cdp_run.py b/pcmdi_metrics/utils/cdp_run.py
diff --git a/pcmdi_metrics/utils/pmp_parameter.py b/pcmdi_metrics/utils/pmp_parameter.py
diff --git a/pcmdi_metrics/utils/pmp_parser.py b/pcmdi_metrics/utils/pmp_parser.py
diff --git a/pcmdi_metrics/variability_mode/variability_modes_driver.py b/pcmdi_metrics/variability_mode/variability_modes_driver.py
