Merge pull request #379 from NREL/release/2.1.6

Release/2.1.6

Author: mdeceglie

.github/workflows/nbval.yaml

@@ -25,7 +25,7 @@ jobs:
     - name: Install notebook environment
       run: |
         python -m pip install --upgrade pip wheel
-        pip install -r requirements.txt -r docs/notebook_requirements.txt .[test]
+        pip install --timeout=300 -r requirements.txt -r docs/notebook_requirements.txt .[test]
     - name: Run notebook and check output
       run: |
         # --sanitize-with:  pre-process text to remove irrelevant differences (e.g. warning filepaths)

.github/workflows/pytest.yaml

@@ -40,7 +40,7 @@ jobs:
     - name: Install ${{ matrix.env }}
       run: |
         python -m pip install --upgrade pip wheel
-        pip install ${{ matrix.env }}
+        pip install --timeout=300 ${{ matrix.env }}
     - name: Test with pytest ${{ matrix.env }}
       run: |
         pytest

.github/workflows/requirements.yaml

@@ -26,4 +26,4 @@ jobs:
     - name: Install notebook environment
       run: |
         python -m pip install --upgrade pip wheel
-        pip install -r requirements.txt -r docs/notebook_requirements.txt
+        pip install --timeout=300 -r requirements.txt -r docs/notebook_requirements.txt

docs/TrendAnalysis_example_pvdaq4.ipynb

@@ -11,10 +11,10 @@ defusedxml==0.7.1
 entrypoints==0.2.3
 html5lib==1.0.1
 ipykernel==4.8.2
-ipython==7.16.3
+ipython==8.10.0
 ipython-genutils==0.2.0
 ipywidgets==7.3.0
-jedi==0.12.1
+jedi==0.16.0
 Jinja2==3.0.0
 jsonschema==2.6.0
 jupyter==1.0.0
@@ -32,11 +32,11 @@ nest-asyncio==1.5.5
 notebook==6.4.12
 numexpr==2.8.0
 pandocfilters==1.4.2
-parso==0.3.1
+parso==0.5.2
 pexpect==4.6.0
 pickleshare==0.7.5
 prometheus-client==0.3.0
-prompt-toolkit==3.0.27
+prompt-toolkit==3.0.30
 ptyprocess==0.6.0
 pycparser==2.20
 Pygments==2.7.4

docs/degradation_and_soiling_example.ipynb

@@ -1,5 +1,6 @@
 RdTools Change Log
 ==================
+.. include:: changelog/v2.1.6.rst
 .. include:: changelog/v2.1.5.rst
 .. include:: changelog/v2.1.4.rst
 .. include:: changelog/v2.1.3.rst

docs/degradation_and_soiling_example_pvdaq_4.ipynb

@@ -0,0 +1,37 @@
+**********************
+v2.1.6 (July 31, 2023)
+**********************
+
+Bug Fixes
+---------
+* Fix NonExistentTimeError in :py:func:`rdtools.clearsky_temperature.get_clearsky_tamb`
+  (:issue:`372` :pull:`373`)
+
+Enhancements
+------------
+* :py:func:`rdtools.degradation.degradation_classical_decomposition` now
+  executes significantly faster. (:pull:`371`)
+
+Requirements
+------------
+* Increased the minimum versions of several dependencies: (:pull:`371`)
+  
+  + pandas increased to 1.3.0 (released July 2, 2021)
+  + numpy to 1.17.3 (released October 17, 2019)
+  + statsmodels to 0.11.0 (released February 21, 2020)
+  + scipy to 1.2.0 (released December 17, 2018)
+
+* Add support for pvlib 0.10 (:pull:`378`)
+* Updated notebook requirements (:pull:`360`)
+* Bumps certifi from 2020.12.5 to 2022.12.7 (:pull:`357`)
+
+Testing
+-------
+* Extended pip timeout (:pull:`360`)
+* Updated example notebooks with new figure sizes (:pull:`360`)
+
+Contributors
+------------
+* Michael Deceglie (:ghuser:`mdeceglie`)
+* Bernat Nicolau (:ghuser:`BernatNicolau`)
+* Kevin Anderson (:ghuser:`kandersolar`)

docs/notebook_requirements.txt

@@ -57,7 +57,9 @@ def get_clearsky_tamb(times, latitude, longitude, window_size=40,
         freq_actual = times.freq
 
     dt_daily = pd.date_range(times.date[0] - buffer, times.date[-1] + buffer,
-                             freq='D', tz=times.tz)
+                             freq='D')
+    dt_daily = dt_daily.tz_localize(times.tz, ambiguous='infer',
+                                    nonexistent='shift_forward')
 
     f = h5py.File(filepath, "r")
 

docs/sphinx/source/changelog.rst

@@ -126,17 +126,9 @@ def degradation_classical_decomposition(energy_normalized,
 
     # Compute yearly rolling mean to isolate trend component using
     # moving average
-    it = df.iterrows()
-    energy_ma = []
-    for i, row in it:
-        if row.years - 0.5 >= min(df.years) and \
-           row.years + 0.5 <= max(df.years):
-            roll = df[(df.years <= row.years + 0.5) &
-                      (df.years >= row.years - 0.5)]
-            energy_ma.append(roll.energy_normalized.mean())
-        else:
-            energy_ma.append(np.nan)
-
+    energy_ma = df['energy_normalized'].rolling('365d', center=True).mean()
+    has_full_year = (df['years'] >= df['years'][0] + 0.5) & (df['years'] <= df['years'][-1] - 0.5)
+    energy_ma[~has_full_year] = np.nan
     df['energy_ma'] = energy_ma
 
     # add intercept-constant to the exogeneous variable
@@ -332,10 +324,8 @@ def _mk_test(x, alpha=0.05):
     n = len(x)
 
     # calculate S
-    s = 0
-    for k in range(n - 1):
-        for j in range(k + 1, n):
-            s += np.sign(x[j] - x[k])
+    x = np.array(x)
+    s = np.sum(np.triu(np.sign(-np.subtract.outer(x, x)), 1))
 
     # calculate the unique data
     unique_x = np.unique(x)

docs/sphinx/source/changelog/v2.1.6.rst

@@ -407,11 +407,11 @@ def test_srr_soiling(soiling_analysis_sensor):
     ci = srr_results['sratio_confidence_interval']
     renorm_factor = srr_results['calc_info']['renormalizing_factor']
     print(f'soiling ci:{ci}')
-    assert 0.965 == pytest.approx(sratio, abs=1e-3),\
+    assert 0.965 == pytest.approx(sratio, abs=1e-3), \
         'Soiling ratio different from expected value in TrendAnalysis.srr_soiling'
-    assert [0.96, 0.97] == pytest.approx(ci, abs=1e-2),\
+    assert [0.96, 0.97] == pytest.approx(ci, abs=1e-2), \
         'Soiling confidence interval different from expected value in TrendAnalysis.srr_soiling'
-    assert 0.974 == pytest.approx(renorm_factor, abs=1e-3),\
+    assert 0.974 == pytest.approx(renorm_factor, abs=1e-3), \
         'Renormalization factor different from expected value in TrendAnalysis.srr_soiling'
 
 

docs/system_availability_example.ipynb

@@ -1,6 +1,8 @@
 import pytest
+import datetime
 import pandas as pd
 
+
 from rdtools.clearsky_temperature import get_clearsky_tamb
 
 
@@ -35,3 +37,21 @@ def test_not_on_land():
     with pytest.warns(UserWarning, match='possibly invalid Lat/Lon coordinates'):
         ocean_cs_tamb = get_clearsky_tamb(dt, 40, -60)
     assert ocean_cs_tamb.isnull().all()
+
+
+def test_with_tricky_timezones():
+    # Some timezones have DST shifts at midnight, which
+    # can lead to NonExistentTimeError. This tests for the
+    # problem in issue #372
+
+    tz = 'America/Santiago'
+    start_date = datetime.datetime(2018, 8, 10, 0, 0, 0)
+    end_date = datetime.datetime(2018, 8, 14, 23, 0, 0)
+    freq = 'H'
+    lat = -24
+    lon = -70
+
+    times = pd.date_range(start=start_date, end=end_date, freq=freq)
+    times = times.tz_localize(tz=tz, ambiguous='infer',
+                              nonexistent='shift_forward')
+    get_clearsky_tamb(times, lat, lon)

rdtools/clearsky_temperature.py

@@ -13,17 +13,17 @@ def test_soiling_srr(soiling_normalized_daily, soiling_insolation, soiling_times
     reps = 10
     np.random.seed(1977)
     sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=reps)
-    assert 0.964369 == pytest.approx(sr, abs=1e-6),\
+    assert 0.964369 == pytest.approx(sr, abs=1e-6), \
         'Soiling ratio different from expected value'
-    assert np.array([0.962540, 0.965295]) == pytest.approx(sr_ci, abs=1e-6),\
+    assert np.array([0.962540, 0.965295]) == pytest.approx(sr_ci, abs=1e-6), \
         'Confidence interval different from expected value'
-    assert 0.960205 == pytest.approx(soiling_info['exceedance_level'], abs=1e-6),\
+    assert 0.960205 == pytest.approx(soiling_info['exceedance_level'], abs=1e-6), \
         'Exceedance level different from expected value'
-    assert 0.984079 == pytest.approx(soiling_info['renormalizing_factor'], abs=1e-6),\
+    assert 0.984079 == pytest.approx(soiling_info['renormalizing_factor'], abs=1e-6), \
         'Renormalizing factor different from expected value'
-    assert len(soiling_info['stochastic_soiling_profiles']) == reps,\
+    assert len(soiling_info['stochastic_soiling_profiles']) == reps, \
         'Length of soiling_info["stochastic_soiling_profiles"] different than expected'
-    assert isinstance(soiling_info['stochastic_soiling_profiles'], list),\
+    assert isinstance(soiling_info['stochastic_soiling_profiles'], list), \
         'soiling_info["stochastic_soiling_profiles"] is not a list'
 
     # Check soiling_info['soiling_interval_summary']
@@ -33,12 +33,12 @@ def test_soiling_srr(soiling_normalized_daily, soiling_insolation, soiling_times
     actual_summary_columns = soiling_info['soiling_interval_summary'].columns.values
 
     for x in actual_summary_columns:
-        assert x in expected_summary_columns,\
+        assert x in expected_summary_columns, \
             f"'{x}' not an expected column in soiling_info['soiling_interval_summary']"
     for x in expected_summary_columns:
-        assert x in actual_summary_columns,\
+        assert x in actual_summary_columns, \
             f"'{x}' was expected as a column, but not in soiling_info['soiling_interval_summary']"
-    assert isinstance(soiling_info['soiling_interval_summary'], pd.DataFrame),\
+    assert isinstance(soiling_info['soiling_interval_summary'], pd.DataFrame), \
         'soiling_info["soiling_interval_summary"] not a dataframe'
     expected_means = pd.Series({'soiling_rate': -0.002644544,
                                 'soiling_rate_low': -0.002847504,
@@ -57,9 +57,9 @@ def test_soiling_srr(soiling_normalized_daily, soiling_insolation, soiling_times
     pd.testing.assert_index_equal(soiling_info['soiling_ratio_perfect_clean'].index, soiling_times,
                                   check_names=False)
     sr_mean = soiling_info['soiling_ratio_perfect_clean'].mean()
-    assert 0.968265 == pytest.approx(sr_mean, abs=1e-6),\
+    assert 0.968265 == pytest.approx(sr_mean, abs=1e-6), \
         "The mean of soiling_info['soiling_ratio_perfect_clean'] differs from expected"
-    assert isinstance(soiling_info['soiling_ratio_perfect_clean'], pd.Series),\
+    assert isinstance(soiling_info['soiling_ratio_perfect_clean'], pd.Series), \
         'soiling_info["soiling_ratio_perfect_clean"] not a pandas series'
 
 
@@ -74,7 +74,7 @@ def test_soiling_srr_consecutive_invalid(soiling_normalized_daily, soiling_insol
     np.random.seed(1977)
     sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=reps,
                                           max_relative_slope_error=20.0, method=method)
-    assert expected_sr == pytest.approx(sr, abs=1e-6),\
+    assert expected_sr == pytest.approx(sr, abs=1e-6), \
         f'Soiling ratio different from expected value for {method} with consecutive invalid intervals'  # noqa: E501
 
 
@@ -96,7 +96,7 @@ def test_soiling_srr_with_precip(soiling_normalized_daily, soiling_insolation, s
     np.random.seed(1977)
     sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation,
                                           clean_criterion=clean_criterion, **kwargs)
-    assert expected_sr == pytest.approx(sr, abs=1e-6),\
+    assert expected_sr == pytest.approx(sr, abs=1e-6), \
         f"Soiling ratio with clean_criterion='{clean_criterion}' different from expected"
 
 
@@ -105,9 +105,9 @@ def test_soiling_srr_confidence_levels(soiling_normalized_daily, soiling_insolat
     np.random.seed(1977)
     sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation,
                                           confidence_level=95, reps=10, exceedance_prob=80.0)
-    assert np.array([0.959322, 0.966066]) == pytest.approx(sr_ci, abs=1e-6),\
+    assert np.array([0.959322, 0.966066]) == pytest.approx(sr_ci, abs=1e-6), \
         'Confidence interval with confidence_level=95 different than expected'
-    assert 0.962691 == pytest.approx(soiling_info['exceedance_level'], abs=1e-6),\
+    assert 0.962691 == pytest.approx(soiling_info['exceedance_level'], abs=1e-6), \
         'soiling_info["exceedance_level"] different than expected when exceedance_prob=80'
 
 
@@ -125,7 +125,7 @@ def test_soiling_srr_clean_threshold(soiling_normalized_daily, soiling_insolatio
     np.random.seed(1977)
     sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=10,
                                           clean_threshold=0.01)
-    assert 0.964369 == pytest.approx(sr, abs=1e-6),\
+    assert 0.964369 == pytest.approx(sr, abs=1e-6), \
         'Soiling ratio with specified clean_threshold different from expected value'
 
     with pytest.raises(NoValidIntervalError):
@@ -139,9 +139,9 @@ def test_soiling_srr_trim(soiling_normalized_daily, soiling_insolation):
     sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=10,
                                           trim=True)
 
-    assert 0.978093 == pytest.approx(sr, abs=1e-6),\
+    assert 0.978093 == pytest.approx(sr, abs=1e-6), \
         'Soiling ratio with trim=True different from expected value'
-    assert len(soiling_info['soiling_interval_summary']) == 1,\
+    assert len(soiling_info['soiling_interval_summary']) == 1, \
         'Wrong number of soiling intervals found with trim=True'
 
 
@@ -153,7 +153,7 @@ def test_soiling_srr_method(soiling_normalized_daily, soiling_insolation, method
     np.random.seed(1977)
     sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=10,
                                           method=method)
-    assert expected_sr == pytest.approx(sr, abs=1e-6),\
+    assert expected_sr == pytest.approx(sr, abs=1e-6), \
         f'Soiling ratio with method="{method}" different from expected value'
 
 
@@ -174,9 +174,9 @@ def test_soiling_srr_recenter_false(soiling_normalized_daily, soiling_insolation
     np.random.seed(1977)
     sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=10,
                                           recenter=False)
-    assert 1 == soiling_info['renormalizing_factor'],\
+    assert 1 == soiling_info['renormalizing_factor'], \
         'Renormalizing factor != 1 with recenter=False'
-    assert 0.966387 == pytest.approx(sr, abs=1e-6),\
+    assert 0.966387 == pytest.approx(sr, abs=1e-6), \
         'Soiling ratio different than expected when recenter=False'
 
 
@@ -188,11 +188,11 @@ def test_soiling_srr_negative_step(soiling_normalized_daily, soiling_insolation)
     with pytest.warns(UserWarning, match='20% or more of the daily data'):
         sr, sr_ci, soiling_info = soiling_srr(stepped_daily, soiling_insolation, reps=10)
 
-    assert list(soiling_info['soiling_interval_summary']['valid'].values) == [True, False, True],\
+    assert list(soiling_info['soiling_interval_summary']['valid'].values) == [True, False, True], \
         'Soiling interval validity differs from expected when a large negative step\
         is incorporated into the data'
 
-    assert 0.936932 == pytest.approx(sr, abs=1e-6),\
+    assert 0.936932 == pytest.approx(sr, abs=1e-6), \
         'Soiling ratio different from expected when a large negative step is incorporated into the data'  # noqa: E501
 
 
@@ -202,10 +202,10 @@ def test_soiling_srr_max_negative_slope_error(soiling_normalized_daily, soiling_
         sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation,
                                               reps=10, max_relative_slope_error=45.0)
 
-    assert list(soiling_info['soiling_interval_summary']['valid'].values) == [True, True, False],\
+    assert list(soiling_info['soiling_interval_summary']['valid'].values) == [True, True, False], \
         'Soiling interval validity differs from expected when max_relative_slope_error=45.0'
 
-    assert 0.958761 == pytest.approx(sr, abs=1e-6),\
+    assert 0.958761 == pytest.approx(sr, abs=1e-6), \
         'Soiling ratio different from expected when max_relative_slope_error=45.0'
 
 
@@ -220,14 +220,14 @@ def test_soiling_srr_with_nan_interval(soiling_normalized_daily, soiling_insolat
     np.random.seed(1977)
     with pytest.warns(UserWarning, match='20% or more of the daily data'):
         sr, sr_ci, soiling_info = soiling_srr(normalized_corrupt, soiling_insolation, reps=reps)
-    assert 0.948792 == pytest.approx(sr, abs=1e-6),\
+    assert 0.948792 == pytest.approx(sr, abs=1e-6), \
         'Soiling ratio different from expected value when an entire interval was NaN'
 
 
 def test_soiling_srr_outlier_factor(soiling_normalized_daily, soiling_insolation):
     _, _, info = soiling_srr(soiling_normalized_daily, soiling_insolation,
                              reps=1, outlier_factor=8)
-    assert len(info['soiling_interval_summary']) == 2,\
+    assert len(info['soiling_interval_summary']) == 2, \
         'Increasing the outlier_factor did not result in the expected number of soiling intervals'
 
 
@@ -256,7 +256,7 @@ def test_soiling_srr_min_interval_length_default(soiling_normalized_daily, soili
     np.random.seed(1977)
     sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily[start:],
                                           soiling_insolation[start:], reps=reps)
-    assert expected_sr == pytest.approx(sr, abs=1e-6),\
+    assert expected_sr == pytest.approx(sr, abs=1e-6), \
         'Soiling ratio different from expected value'
 
 

rdtools/degradation.py

@@ -1,10 +1,10 @@
 h5py==2.8.0
 matplotlib==3.0.0
-numpy==1.15
-pandas==0.23.2
+numpy==1.17.3
+pandas==1.3.0
 pvlib==0.7.0
-scipy==1.1.0
-statsmodels==0.9.0
+scipy==1.2.0
+statsmodels==0.11.0
 tables==3.5.1
 numexpr==2.7.1  # https://github.com/pydata/numexpr/issues/369
 plotly==4.0.0

rdtools/test/analysis_chains_test.py

@@ -1,5 +1,5 @@
 cached-property==1.5.2
-certifi==2020.12.5
+certifi==2022.12.7
 chardet==4.0.0
 cycler==0.10.0
 fonttools==4.37.1