Merge pull request #4629 from springfall2008/fix/octopus-url-current-rates-2726

feat(octopus): flag Octopus rate URLs with no current/future rates

Author: springfall2008

apps/predbat/octopus.py

@@ -2342,6 +2342,19 @@ def download_octopus_rates(self, url):
             else:
                 raise ValueError
 
+        # Downloaded data with a valid response but nothing covering the current time onward -
+        # e.g. a retired Octopus product whose URL still returns historical results (#2726).
+        # Retrying wouldn't help (the URL isn't broken, the product behind it is stale), so this
+        # is checked once per fresh download rather than inside the retry loop above.
+        if max(pdata.keys(), default=-1) < self.minutes_now:
+            self.log("Warn: Octopus: Downloaded data from URL {} has no current or future rates - the product may have been retired, check the URL in apps.yaml".format(url))
+            self.record_status("Warn: Octopus: URL has no current rates, check apps.yaml", debug=url, had_errors=True)
+            if url in self.octopus_url_cache:
+                pdata = self.octopus_url_cache[url]["data"]
+                return pdata
+            else:
+                raise ValueError
+
         # Cache New Octopus data
         self.octopus_url_cache[url] = {}
         self.octopus_url_cache[url]["stamp"] = now

apps/predbat/tests/test_octopus_download_rates.py

@@ -28,6 +28,8 @@ def test_octopus_download_rates(my_predbat):
     10. download_octopus_rates_func - HTTP error status
     11. download_octopus_rates_func - JSON decode error
     12. download_octopus_rates_func - missing 'results' key
+    13. No current/future rates (#2726) - falls back to stale cache when available
+    14. No current/future rates (#2726) - raises ValueError when no cache available
     """
     print("\n=== Test Octopus download_octopus_rates ===")
     failed = False
@@ -38,6 +40,11 @@ def test_octopus_download_rates(my_predbat):
     my_predbat.midnight_utc = datetime.strptime("2024-06-12T00:00:00+00:00", "%Y-%m-%dT%H:%M:%S%z")
     my_predbat.debug_enable = False
     my_predbat.failures_total = 0
+    # All the mock rate data below starts at minute 0 and covers forward from there - pin "now"
+    # to 0 so the #2726 "has current rates" check (added below, tested explicitly in Test 13/14)
+    # passes for scenarios that aren't about that check, regardless of real wall-clock time or
+    # whatever a previous test left minutes_now as.
+    my_predbat.minutes_now = 0
 
     # Mock the download_octopus_rates_func to return rate data
     mock_rate_data = {
@@ -195,6 +202,50 @@ def test_octopus_download_rates(my_predbat):
     else:
         print("✓ Test 6 passed - Retry mechanism works correctly")
 
+    # Test 13: No current/future rates (#2726) - falls back to stale cache when available.
+    # A retired Octopus product's URL can still return a valid 200/JSON response with genuine
+    # historical results, so this can't be caught by any of the existing failure paths above -
+    # it needs its own check that the data actually reaches "now" onward.
+    print("\nTest 13: No current/future rates (#2726) - falls back to stale cache")
+    test_url = "https://api.octopus.energy/test-stale-product"
+    stale_cached_data = {0: 8.0, 60: 9.0}
+    my_predbat.octopus_url_cache = {
+        test_url: {
+            "stamp": datetime.now() - timedelta(minutes=50),  # expired, forces a fresh download
+            "midnight_utc": my_predbat.midnight_utc,
+            "data": stale_cached_data,
+        }
+    }
+    my_predbat.minutes_now = 120
+    # Every key is before "now" (120) - looks like a retired product still serving historical data.
+    historical_only_data = {0: 5.0, 60: 6.0}
+
+    with patch.object(my_predbat, 'download_octopus_rates_func', return_value=historical_only_data):
+        result = my_predbat.download_octopus_rates(test_url)
+
+    if result != stale_cached_data:
+        print(f"✗ Test 13 failed - Expected stale cached data {stale_cached_data}, got {result}")
+        failed = True
+    else:
+        print("✓ Test 13 passed - No current rates falls back to stale cache")
+
+    # Test 14: No current/future rates (#2726) - raises ValueError when no cache available
+    print("\nTest 14: No current/future rates (#2726) - raises ValueError when no cache")
+    test_url = "https://api.octopus.energy/test-stale-product-no-cache"
+    my_predbat.octopus_url_cache = {}
+    my_predbat.minutes_now = 120
+    historical_only_data = {0: 5.0, 60: 6.0}
+
+    with patch.object(my_predbat, 'download_octopus_rates_func', return_value=historical_only_data):
+        try:
+            result = my_predbat.download_octopus_rates(test_url)
+            print("✗ Test 14 failed - Expected ValueError to be raised")
+            failed = True
+        except ValueError:
+            print("✓ Test 14 passed - ValueError raised when no current rates and no cache")
+
+    my_predbat.minutes_now = 0  # restore for the remaining tests below
+
     # Test 7: download_octopus_rates_func - successful single page
     print("\nTest 7: download_octopus_rates_func - successful single page")
     my_predbat.debug_enable = False