Adding cdp to pmp utils

kristinchang3 · kristinchang3 · commit 68a935236f37 · 2025-10-29T14:57:48.000-07:00
diff --git a/conda-env/ci.yml b/conda-env/ci.yml
@@ -13,7 +13,6 @@ dependencies:
     - numpy >=2.0.0,<3
     - cartopy >=0.22.0
     - matplotlib >=3.7.1
-    - cdp >=1.7.0
     - eofs >=2.0.0
     - seaborn >=0.12.2
     #- enso_metrics >=1.1.5
diff --git a/conda-env/dev.yml b/conda-env/dev.yml
@@ -13,7 +13,6 @@ dependencies:
     - numpy >=2.0.0,<3
     - cartopy >=0.22.0
     - matplotlib >=3.7.1
-    - cdp >=1.7.0
     - eofs >=2.0.0
     - seaborn >=0.12.2
     #- enso_metrics >=1.1.5
diff --git a/pcmdi_metrics/cloud_feedback/lib/argparse_functions.py b/pcmdi_metrics/cloud_feedback/lib/argparse_functions.py
@@ -1,6 +1,6 @@
 import argparse
 
-from cdp.cdp_parser import CDPParser
+from pcmdi_metrics.utils.cdp_parser import CDPParser
 
 
 def AddParserArgument():
diff --git a/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py b/pcmdi_metrics/diurnal/scripts/compositeDiurnalStatistics.py
@@ -24,7 +24,7 @@
 import multiprocessing as mp
 import os
 
-import cdp
+from pcmdi_metrics.utils import cdp_run
 import cftime
 import numpy as np
 import xarray as xr
@@ -252,7 +252,7 @@ def compute(params):
     print("FILES:", fileList)
     params = [INPUT(args, name, template) for name in fileList]
     print("PARAMS:", params)
-    cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
 
 def add_one_month(t):
diff --git a/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py b/pcmdi_metrics/diurnal/scripts/computeStdOfDailyMeans.py
@@ -16,7 +16,7 @@
 import multiprocessing as mp
 import os
 
-import cdp
+from pcmdi_metrics.utils import cdp_run
 import cftime
 import xarray as xr
 
@@ -195,7 +195,7 @@ def compute(params, debug=False):
     params = [INPUT(args, name, template) for name in fileList]
     print("PARAMS:", params)
 
-    cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
 
 def add_one_month(t):
diff --git a/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py b/pcmdi_metrics/diurnal/scripts/std_of_dailymeans.py
@@ -24,7 +24,7 @@
 import multiprocessing as mp
 import os
 
-import cdp
+from pcmdi_metrics.utils import cdp_run
 import numpy as np
 
 import pcmdi_metrics
@@ -167,7 +167,7 @@ def compute(param):
     params = [INPUT(args, name, template) for name in files]
     print("PARAMS:", params)
 
-    results = cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    results = cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
     for r in results:
         m, region, res = r
diff --git a/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py b/pcmdi_metrics/diurnal/scripts/std_of_hourlyvalues.py
@@ -21,7 +21,7 @@
 import multiprocessing as mp
 import os
 
-import cdp
+from pcmdi_metrics.utils import cdp_run
 
 import pcmdi_metrics
 from pcmdi_metrics import resources
@@ -172,7 +172,7 @@ def compute(param):
     params = [INPUT(args, name, template) for name in files]
     print("PARAMS:", params)
 
-    results = cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    results = cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
     for r in results:
         m, region, res = r
diff --git a/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py b/pcmdi_metrics/diurnal/scripts/std_of_meandiurnalcycle.py
@@ -19,7 +19,7 @@
 import multiprocessing as mp
 import os
 
-import cdp
+from pcmdi_metrics.utils import cdp_run
 
 import pcmdi_metrics
 from pcmdi_metrics import resources
@@ -182,7 +182,7 @@ def compute(param):
     params = [INPUT(args, name, template) for name in files]
     print("PARAMS:", params)
 
-    results = cdp.cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
+    results = cdp_run.multiprocess(compute, params, num_workers=args.num_workers)
 
     for r in results:
         m, region, res = r
diff --git a/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py b/pcmdi_metrics/drcdm/lib/create_drcdm_parser.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from pcmdi_metrics.mean_climate.lib import pmp_parser
+from pcmdi_metrics.utils import pmp_parser
 
 
 def create_extremes_parser():
diff --git a/pcmdi_metrics/enso/lib/enso_lib.py b/pcmdi_metrics/enso/lib/enso_lib.py
@@ -7,7 +7,7 @@
 from collections import defaultdict
 
 import pcmdi_metrics
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser
+from pcmdi_metrics.utils.pmp_parser import PMPParser
 
 
 def AddParserArgument():
diff --git a/pcmdi_metrics/io/__init__.py b/pcmdi_metrics/io/__init__.py
@@ -2,6 +2,7 @@
 from .xcdat_openxml import xcdat_open  # noqa  # isort:skip
 from .string_constructor import StringConstructor, fill_template  # noqa  # isort:skip
 from . import base  # noqa
+from . import cdp_io # noqa
 
 from .xcdat_dataset_io import (  # noqa  # isort:skip
     da_to_ds,
diff --git a/pcmdi_metrics/io/base.py b/pcmdi_metrics/io/base.py
@@ -11,13 +11,12 @@
 from datetime import datetime
 from subprocess import PIPE, Popen
 
-import cdp.cdp_io
 import numpy
 import requests
 
 import pcmdi_metrics
 from pcmdi_metrics import LOG_LEVEL
-from pcmdi_metrics.io import StringConstructor
+from pcmdi_metrics.io import cdp_io, StringConstructor
 
 logging.getLogger("pcmdi_metrics").setLevel(LOG_LEVEL)  # set up to log errors
 
@@ -264,7 +263,7 @@ def alphanum(key):
 # ----------
 
 
-class Base(cdp.cdp_io.CDPIO, StringConstructor):
+class Base(cdp_io.CDPIO, StringConstructor):
     def __init__(self, root, file_template, file_mask_template=None):
         StringConstructor.__init__(self, root + "/" + file_template)
         self.root = root
diff --git a/pcmdi_metrics/io/cdp_io.py b/pcmdi_metrics/io/cdp_io.py
@@ -0,0 +1,16 @@
+from __future__ import print_function
+
+import abc
+from six import with_metaclass
+
+
+class CDPIO(with_metaclass(abc.ABCMeta, object)):
+    @abc.abstractmethod
+    def read(self):
+        """Read a file."""
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def write(self):
+        """Write a file."""
+        raise NotImplementedError()
diff --git a/pcmdi_metrics/mean_climate/lib/create_mean_climate_parser.py b/pcmdi_metrics/mean_climate/lib/create_mean_climate_parser.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 import ast
 
-from pcmdi_metrics.mean_climate.lib import pmp_parser
+from pcmdi_metrics.utils import pmp_parser
 
 
 def create_mean_climate_parser():
diff --git a/pcmdi_metrics/mean_climate/pcmdi_compute_climatologies.py b/pcmdi_metrics/mean_climate/pcmdi_compute_climatologies.py
@@ -4,7 +4,7 @@
 
 from pcmdi_metrics.io import StringConstructor
 from pcmdi_metrics.mean_climate.lib import calculate_climatology
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPMetricsParser
+from pcmdi_metrics.utils.pmp_parser import PMPMetricsParser
 
 ver = datetime.datetime.now().strftime("v%Y%m%d")
 
diff --git a/pcmdi_metrics/mjo/mjo_metrics_driver.py b/pcmdi_metrics/mjo/mjo_metrics_driver.py
@@ -42,7 +42,7 @@
 from shutil import copyfile
 
 import pcmdi_metrics
-from pcmdi_metrics.mean_climate.lib import pmp_parser
+from pcmdi_metrics.utils import pmp_parser
 from pcmdi_metrics.mjo.lib import (
     AddParserArgument,
     YearCheck,
diff --git a/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py b/pcmdi_metrics/monsoon_sperber/driver_monsoon_sperber.py
@@ -54,7 +54,7 @@
 
 from pcmdi_metrics.io import load_regions_specs, region_subset, xcdat_open
 from pcmdi_metrics.io.base import Base
-from pcmdi_metrics.mean_climate.lib import pmp_parser
+from pcmdi_metrics.utils import pmp_parser
 from pcmdi_metrics.monsoon_sperber.lib import (
     AddParserArgument,
     YearCheck,
diff --git a/pcmdi_metrics/monsoon_wang/lib/argparse_functions.py b/pcmdi_metrics/monsoon_wang/lib/argparse_functions.py
@@ -1,4 +1,4 @@
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser
+from pcmdi_metrics.utils.pmp_parser import PMPParser
 
 
 def create_monsoon_wang_parser():
diff --git a/pcmdi_metrics/precip_distribution/precip_distribution_driver.py b/pcmdi_metrics/precip_distribution/precip_distribution_driver.py
@@ -7,7 +7,7 @@
 import xarray as xr
 
 from pcmdi_metrics.io import StringConstructor, get_calendar, xcdat_open
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser
+from pcmdi_metrics.utils.pmp_parser import PMPParser
 from pcmdi_metrics.precip_distribution.lib import (
     AddParserArgument,
     Regrid_xr,
diff --git a/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py b/pcmdi_metrics/precip_variability/variability_across_timescales_PS_driver.py
@@ -2,7 +2,7 @@
 import glob
 import os
 
-from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser
+from pcmdi_metrics.utils.pmp_parser import PMPParser
 from pcmdi_metrics.precip_variability.lib import (
     AddParserArgument,
     precip_variability_across_timescale,
diff --git a/pcmdi_metrics/sea_ice/lib/sea_ice_parser.py b/pcmdi_metrics/sea_ice/lib/sea_ice_parser.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from pcmdi_metrics.mean_climate.lib import pmp_parser
+from pcmdi_metrics.utils import pmp_parser
 
 
 def create_sea_ice_parser():
diff --git a/pcmdi_metrics/utils/__init__.py b/pcmdi_metrics/utils/__init__.py
@@ -31,3 +31,7 @@
 from .string_constructor import StringConstructor, fill_template
 from .tree_dict import tree
 from .xr_to_cdms2 import cdms2_to_xarray, xarray_to_cdms2
+from .cdp_parser import CDPParser
+from .cdp_parameter import CDPParameter
+from .pmp_parser import PMPParser, PMPMetricsParser
+from .pmp_parameter import PMPParameter, PMPMetricsParameter
diff --git a/pcmdi_metrics/utils/cdp_parameter.py b/pcmdi_metrics/utils/cdp_parameter.py
@@ -0,0 +1,69 @@
+from __future__ import print_function
+
+import abc
+import importlib
+import sys
+import os
+import copy
+import types
+from six import with_metaclass
+
+
+class CDPParameter(object):
+    def __add__(self, other):
+        """
+        Deepcopy any attribute of `other` into self.
+        """
+        # First make a deepcopy of the current object.
+        duplicate = copy.deepcopy(self)
+        
+        for attr in dir(other):
+            # Ignore any of the hidden attributes.
+            if attr.startswith('_') or \
+                isinstance(getattr(other, attr), types.MethodType):
+                continue
+
+            val = copy.deepcopy(getattr(other, attr))
+            setattr(duplicate, attr, val)
+        
+        return duplicate
+
+    def check_values(self):
+        """
+        Check that all of the variables in
+        this parameter file are valid.
+        """
+        pass
+
+    def load_parameter_from_py(self, parameter_file_path):
+        """
+        Initialize a parameter object from a Python script.
+        """
+        parameter_as_module = \
+            self.import_user_parameter_file_as_module(parameter_file_path)
+        self.load_parameters_from_module(parameter_as_module)
+
+    def import_user_parameter_file_as_module(self, parameter_file_path):
+        if not os.path.isfile(parameter_file_path):
+            raise IOError('Parameter file %s not found.' % parameter_file_path)
+
+        path_to_module = os.path.split(parameter_file_path)[0]
+        module_name = os.path.split(parameter_file_path)[1]
+        if module_name.count('.') > 1:
+            raise ValueError("Filename cannot contain '.' outside extension.")
+        if '.' in module_name:
+            module_name = module_name.split('.')[0]
+
+        sys.path.insert(0, path_to_module)
+        return importlib.import_module(module_name)
+
+    def load_parameters_from_module(self, parameter_as_module):
+        user_defined_parameters = []
+        for user_parameter in dir(parameter_as_module):
+            if not user_parameter.startswith('__'):
+                user_defined_parameters.append(user_parameter)
+
+        # Initialize the variables in this parameter, so the driver can
+        # access them as if they were defined regularly.
+        for p in user_defined_parameters:
+            self.__dict__[p] = getattr(parameter_as_module, p)
diff --git a/pcmdi_metrics/utils/cdp_parser.py b/pcmdi_metrics/utils/cdp_parser.py
diff --git a/pcmdi_metrics/utils/cdp_run.py b/pcmdi_metrics/utils/cdp_run.py
diff --git a/pcmdi_metrics/utils/pmp_parameter.py b/pcmdi_metrics/utils/pmp_parameter.py
diff --git a/pcmdi_metrics/utils/pmp_parser.py b/pcmdi_metrics/utils/pmp_parser.py
diff --git a/pcmdi_metrics/variability_mode/variability_modes_driver.py b/pcmdi_metrics/variability_mode/variability_modes_driver.py
