Merge pull request #156 from pysat/rc_v0.2.1

Release Candidate v0.2.1

Author: aburrell

.github/workflows/main.yml

@@ -26,8 +26,8 @@ jobs:
             os: ubuntu-latest
             test_config: "NEP29"
           # Operational compliance settings
-          - python-version: "3.6.8"
-            numpy_ver: "1.19.5"
+          - python-version: "3.9"
+            numpy_ver: "1.23.5"
             os: "ubuntu-20.04"
             test_config: "Ops"
 
@@ -44,9 +44,7 @@ jobs:
       if: ${{ matrix.test_config == 'Ops'}}
       run: |
         pip install numpy==${{ matrix.numpy_ver }}
-        pip install -r requirements.txt
-        pip install -r test_requirements.txt
-        pip install .
+        pip install --upgrade-strategy only-if-needed .[test]
 
     - name: Install NEP29 dependencies
       if: ${{ matrix.test_config == 'NEP29'}}

CHANGELOG.md

@@ -3,6 +3,17 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/).
 
+[0.2.1] - 2024-11-18
+--------------------
+* Enhancements
+  * Added a utility function for evaluating fill values of different types
+* Maintenance
+  * Updated Ops tests to new lower limit of Python 3.9 and removed 3.6 support
+* Bugs
+  * Fixed error in mock downloading F10.7 prelim files
+  * Fixed combine_kp to consider desired time limits and fill values when
+    loading the standard dataset
+
 [0.2.0] - 2024-08-30
 --------------------
 * Enhancements

README.md

@@ -22,7 +22,7 @@ give some examples on how to use the routines.
 
 pysatSpaceWeather uses common Python modules, as well as modules developed by
 and for the Space Physics community.  This module officially supports
-Python 3.7+.
+Python 3.9+.
 
 | Common modules | Community modules |
 | -------------- | ----------------- |

docs/installation.rst

@@ -19,7 +19,7 @@ Prerequisites
 
 pysatSpaceWeather uses common Python modules, as well as modules developed by
 and for the Space Physics community.  This module officially supports
-Python 3.6 and 3.9+.
+Python 3.9+.
 
  ============== =================
  Common modules Community modules

pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pysatSpaceWeather"
-version = "0.2.0"
+version = "0.2.1"
 description = 'pysat support for Space Weather Indices'
 readme = "README.md"
-requires-python = ">=3.6"
+requires-python = ">=3.9"
 license = {file = "LICENSE"}
 authors = [
     {name = "Angeline Burrell", email = "pysat.developers@gmail.com"}
@@ -21,7 +21,7 @@ classifiers = [
   "License :: OSI Approved :: BSD License",
   "Natural Language :: English",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.6",
+  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",

pysatSpaceWeather/instruments/methods/f107.py

@@ -17,6 +17,7 @@
 import pysat
 
 import pysatSpaceWeather as pysat_sw
+from pysatSpaceWeather.instruments.methods.general import is_fill_val
 
 
 def acknowledgements(tag):
@@ -162,7 +163,6 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None):
 
     # Cycle through the desired time range
     itime = dt.datetime(start.year, start.month, start.day)
-
     while itime < stop and inst_flag is not None:
         # Load and save the standard data for as many times as possible
         if inst_flag == 'standard':
@@ -193,8 +193,13 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None):
                     fill_val = f107_inst.meta['f107'][
                         f107_inst.meta.labels.fill_val]
 
-                good_vals = standard_inst['f107'][good_times] != fill_val
+                good_vals = np.array([not is_fill_val(val, fill_val) for val
+                                      in standard_inst['f107'][good_times]])
                 new_times = list(standard_inst.index[good_times][good_vals])
+            else:
+                new_times = []
+
+            if len(new_times) > 0:
                 f107_times.extend(new_times)
                 new_vals = list(standard_inst['f107'][good_times][good_vals])
                 f107_values.extend(new_vals)
@@ -237,12 +242,14 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None):
                     # Get the good times and values
                     good_times = ((forecast_inst.index >= itime)
                                   & (forecast_inst.index < stop))
-                    good_vals = forecast_inst['f107'][good_times] != fill_val
+                    good_vals = np.array([
+                        not is_fill_val(val, fill_val) for val
+                        in forecast_inst['f107'][good_times]])
 
                 # Save desired data and cycle time
                 if len(good_vals) > 0:
-                    new_times = list(
-                        forecast_inst.index[good_times][good_vals])
+                    new_times = list(forecast_inst.index[good_times][
+                        good_vals])
                     f107_times.extend(new_times)
                     new_vals = list(
                         forecast_inst['f107'][good_times][good_vals])
@@ -267,8 +274,6 @@ def combine_f107(standard_inst, forecast_inst, start=None, stop=None):
     if len(f107_times) == 0:
         f107_times = date_range
 
-    date_range = pds.date_range(start=start, end=end_date, freq=freq)
-
     if date_range[0] < f107_times[0]:
         # Extend the time and value arrays from their beginning with fill
         # values

pysatSpaceWeather/instruments/methods/general.py

@@ -17,6 +17,38 @@
 import pysat
 
 
+def is_fill_val(data, fill_val):
+    """Evaluate whether or not a value is a fill value.
+
+    Parameters
+    ----------
+    data : int, float, or str
+        Data value
+    fill_val : int, float, or str
+        Fill value
+
+    Returns
+    -------
+    is_fill : bool
+        True if the data is equal to the fill value, False if it is not.
+
+    """
+
+    try:
+        # NaN and finite evaluation will fail for non-numeric types
+        if np.isnan(fill_val):
+            is_fill = np.isnan(data)
+        elif np.isfinite(fill_val):
+            is_fill = data == fill_val
+        else:
+            is_fill = ~np.isfinite(data)
+    except TypeError:
+        # Use equality for string and similar types
+        is_fill = data == fill_val
+
+    return is_fill
+
+
 def preprocess(inst):
     """Preprocess the meta data by replacing the file fill values with NaN.
 

pysatSpaceWeather/instruments/methods/kp_ap.py

@@ -16,6 +16,7 @@
 import pysat
 
 import pysatSpaceWeather as pysat_sw
+from pysatSpaceWeather.instruments.methods import general
 from pysatSpaceWeather.instruments.methods import gfz
 from pysatSpaceWeather.instruments.methods import swpc
 
@@ -600,7 +601,9 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None,
     while itime < stop and inst_flag is not None:
         # Load and save the standard data for as many times as possible
         if inst_flag == 'standard':
-            standard_inst.load(date=itime)
+            # Test to see if data loading is needed
+            if not np.any(standard_inst.index == itime):
+                standard_inst.load(date=itime)
 
             if notes.find("standard") < 0:
                 notes += " the {:} source ({:} to ".format(inst_flag,
@@ -610,9 +613,23 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None,
                 inst_flag = 'forecast' if recent_inst is None else 'recent'
                 notes += "{:})".format(itime.date())
             else:
-                kp_times.extend(list(standard_inst.index))
-                kp_values.extend(list(standard_inst['Kp']))
-                itime = kp_times[-1] + pds.DateOffset(hours=3)
+                local_fill_val = standard_inst.meta[
+                    'Kp', standard_inst.meta.labels.fill_val]
+                good_times = ((standard_inst.index >= itime)
+                              & (standard_inst.index < stop))
+                good_vals = np.array([
+                    not general.is_fill_val(val, local_fill_val)
+                    for val in standard_inst['Kp'][good_times]])
+                new_times = list(standard_inst.index[good_times][good_vals])
+
+                if len(new_times) > 0:
+                    kp_times.extend(new_times)
+                    kp_values.extend(list(
+                        standard_inst['Kp'][good_times][good_vals]))
+                    itime = kp_times[-1] + pds.DateOffset(hours=3)
+                else:
+                    inst_flag = 'forecast' if recent_inst is None else 'recent'
+                    notes += "{:})".format(itime.date())
 
         # Load and save the recent data for as many times as possible
         if inst_flag == 'recent':
@@ -637,18 +654,20 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None,
 
                 # Determine which times to save
                 if recent_inst.empty:
-                    good_vals = []
+                    new_times = []
                 else:
                     local_fill_val = recent_inst.meta[
                         'Kp', recent_inst.meta.labels.fill_val]
                     good_times = ((recent_inst.index >= itime)
                                   & (recent_inst.index < stop))
-                    good_vals = recent_inst['Kp'][good_times] != local_fill_val
+                    good_vals = np.array([
+                        not general.is_fill_val(val, local_fill_val)
+                        for val in recent_inst['Kp'][good_times]])
+                    new_times = list(recent_inst.index[good_times][good_vals])
 
                 # Save output data and cycle time
-                if len(good_vals):
-                    kp_times.extend(list(
-                        recent_inst.index[good_times][good_vals]))
+                if len(new_times) > 0:
+                    kp_times.extend(new_times)
                     kp_values.extend(list(
                         recent_inst['Kp'][good_times][good_vals]))
                     itime = kp_times[-1] + pds.DateOffset(hours=3)
@@ -683,17 +702,21 @@ def combine_kp(standard_inst=None, recent_inst=None, forecast_inst=None,
                         'Kp', forecast_inst.meta.labels.fill_val]
                     good_times = ((forecast_inst.index >= itime)
                                   & (forecast_inst.index < stop))
-                    good_vals = forecast_inst['Kp'][
-                        good_times] != local_fill_val
+                    good_vals = np.array([
+                        not general.is_fill_val(val, local_fill_val)
+                        for val in forecast_inst['Kp'][good_times]])
 
                     # Save desired data
                     new_times = list(forecast_inst.index[good_times][good_vals])
-                    kp_times.extend(new_times)
-                    new_vals = list(forecast_inst['Kp'][good_times][good_vals])
-                    kp_values.extend(new_vals)
 
-                    # Cycle time
-                    itime = kp_times[-1] + pds.DateOffset(hours=3)
+                    if len(new_times) > 0:
+                        kp_times.extend(new_times)
+                        new_vals = list(forecast_inst['Kp'][good_times][
+                            good_vals])
+                        kp_values.extend(new_vals)
+
+                        # Cycle time
+                        itime = kp_times[-1] + pds.DateOffset(hours=3)
             notes += "{:})".format(itime.date())
 
             inst_flag = None

pysatSpaceWeather/instruments/methods/swpc.py

@@ -240,9 +240,9 @@ def old_indices_dsd_download(name, date_array, data_path, local_files, today,
                 else:
                     # Set the saved filename
                     saved_fname = os.path.join(mock_download_dir, local_fname)
-                    downloaded = True
 
                     if os.path.isfile(saved_fname):
+                        downloaded = True
                         rewritten = True
                     else:
                         pysat.logger.info("".join([saved_fname, "is missing, ",
@@ -273,6 +273,7 @@ def old_indices_dsd_download(name, date_array, data_path, local_files, today,
     # Close connection after downloading all dates
     if mock_download_dir is None:
         ftp.close()
+
     return
 
 

pysatSpaceWeather/tests/test_methods_general.py

@@ -9,6 +9,7 @@
 """Integration and unit test suite for ACE methods."""
 
 import numpy as np
+import pytest
 
 import pysat
 
@@ -22,24 +23,45 @@ def setup_method(self):
         """Create a clean testing setup."""
         self.testInst = pysat.Instrument('pysat', 'testing')
         self.testInst.load(date=self.testInst.inst_module._test_dates[''][''])
+        self.var = self.testInst.variables[0]
         return
 
     def teardown_method(self):
         """Clean up previous testing setup."""
-        del self.testInst
+        del self.testInst, self.var
         return
 
     def test_preprocess(self):
         """Test the preprocessing routine updates all fill values to be NaN."""
 
         # Make sure at least one fill value is not already NaN
-        var = self.testInst.variables[0]
-        self.testInst.meta[var] = {self.testInst.meta.labels.fill_val: 0.0}
+        self.testInst.meta[self.var] = {self.testInst.meta.labels.fill_val: 0.0}
 
         # Update the meta data using the general preprocess routine
         general.preprocess(self.testInst)
 
         # Test the output
         assert np.isnan(
-            self.testInst.meta[var, self.testInst.meta.labels.fill_val])
+            self.testInst.meta[self.var, self.testInst.meta.labels.fill_val])
+        return
+
+    @pytest.mark.parametrize("fill_val", [-1.0, -1, np.nan, np.inf, ''])
+    def test_is_fill(self, fill_val):
+        """Test the successful evaluation of fill values.
+
+        Parameters
+        ----------
+        fill_val : float, int, or str
+            Fill value to use as a comparison
+
+        """
+        # Set the data value to not be a fill value
+        if fill_val != '':
+            self.var = -47
+
+        # Evaluate the variable is False
+        assert not general.is_fill_val(self.var, fill_val)
+
+        # Evaluate the fill value is a fill value
+        assert general.is_fill_val(fill_val, fill_val)
         return