Merge pull request #4030 from springfall2008/feat/open_meteo_backup

Feat, add open meteo backup for forecast.solar

Author: springfall2008

apps/predbat/components.py

@@ -103,6 +103,7 @@
             "solcast_poll_hours": {"required": False, "config": "solcast_poll_hours", "default": 8},
             "forecast_solar": {"required": False, "config": "forecast_solar", "default": False},
             "forecast_solar_max_age": {"required": False, "config": "forecast_solar_max_age", "default": 8},
+            "forecast_solar_open_meteo_backup": {"required": False, "config": "forecast_solar_open_meteo_backup", "default": False},
             "pv_forecast_today": {"required": False, "config": "pv_forecast_today"},
             "pv_forecast_tomorrow": {"required": False, "config": "pv_forecast_tomorrow"},
             "pv_forecast_d3": {"required": False, "config": "pv_forecast_d3"},

apps/predbat/config.py

@@ -2218,6 +2218,7 @@
     "predheat": {"type": "dict"},
     "forecast_solar": {"type": "dict_list"},
     "forecast_solar_max_age": {"type": "float"},
+    "forecast_solar_open_meteo_backup": {"type": "boolean"},
     "open_meteo_forecast": {"type": "dict_list"},
     "open_meteo_forecast_max_age": {"type": "float"},
     "enable_coarse_fine_levels": {"type": "boolean"},

apps/predbat/predbat.py

@@ -36,7 +36,7 @@
 import requests
 import asyncio
 
-THIS_VERSION = "v8.40.3"
+THIS_VERSION = "v8.40.4"
 
 from download import predbat_update_move, predbat_update_download, check_install, resolve_predbat_repository, DEFAULT_PREDBAT_REPOSITORY
 from const import MINUTE_WATT

apps/predbat/solcast.py

@@ -69,6 +69,7 @@ def initialize(
         solcast_poll_hours,
         forecast_solar,
         forecast_solar_max_age,
+        forecast_solar_open_meteo_backup,
         pv_forecast_today,
         pv_forecast_tomorrow,
         pv_forecast_d3,
@@ -84,6 +85,7 @@ def initialize(
         self.solcast_poll_hours = solcast_poll_hours
         self.forecast_solar = forecast_solar
         self.forecast_solar_max_age = forecast_solar_max_age
+        self.forecast_solar_open_meteo_backup = forecast_solar_open_meteo_backup
         self.pv_forecast_today = pv_forecast_today
         self.pv_forecast_tomorrow = pv_forecast_tomorrow
         self.pv_forecast_d3 = pv_forecast_d3
@@ -283,16 +285,18 @@ async def download_open_meteo_ensemble_data(self, lat, lon, tilt, az, kwp, syste
             result[ts] = dp4((gti_p10 / 1000.0) * kwp * (1.0 - system_loss))
         return result
 
-    async def download_open_meteo_data(self):
+    async def download_open_meteo_data(self, configs=None):
         """
         Download Open-Meteo forecast data and convert to PV power estimates.
         Uses GTI (global tilted irradiance) with simple temperature derating for P50,
         and ensemble members for P10. Returns (sorted_data, max_kwh).
+        If configs is provided it is used directly; otherwise self.open_meteo_forecast is used.
         """
         period_data = {}
         max_kwh = 0
 
-        configs = self.open_meteo_forecast
+        if configs is None:
+            configs = self.open_meteo_forecast
         if configs is None:
             raise ValueError("SolarAPI: No Open-Meteo forecast configurations found")
         if not isinstance(configs, list):
@@ -1258,6 +1262,10 @@ async def fetch_pv_forecast(self):
             pv_forecast_data, max_kwh = await self.download_forecast_solar_data()
             divide_by = 30.0
             create_pv10 = True
+            if not pv_forecast_data and self.forecast_solar_open_meteo_backup:
+                self.log("SolarAPI: Forecast Solar returned no data, falling back to Open-Meteo backup")
+                backup_configs = self.open_meteo_forecast if self.open_meteo_forecast else self.forecast_solar
+                pv_forecast_data, max_kwh = await self.download_open_meteo_data(configs=backup_configs)
         elif self.open_meteo_forecast:
             self.log("SolarAPI: Obtaining solar forecast from Open-Meteo API")
             pv_forecast_data, max_kwh = await self.download_open_meteo_data()

apps/predbat/tests/test_random_scenarios.py

@@ -43,19 +43,54 @@
 PV_PEAK_OPTIONS_KW = [0.0, 1.0, 2.0, 4.0, 6.0, 10.0]
 
 RATE_TYPES = ["single", "dual", "triple", "halfhourly"]
+IMPORT_RATE_TYPES = RATE_TYPES + ["negative_halfhourly"]
 
 # ---------------------------------------------------------------------------
 # Daily profile generators  (return list[float] with exactly 1440 elements)
 # ---------------------------------------------------------------------------
 
 
+def _smooth_rate_profile(control_points, allow_negative=False):
+    """Cosine-interpolate between control points to produce a 1440-minute rate profile.
+
+    Adjacent control points are connected with a smooth S-curve (raised cosine), so the
+    profile ramps gradually between each peak/trough rather than jumping abruptly.
+
+    Args:
+        control_points: list of (minute, rate) pairs covering at least minute 0 and 1439
+        allow_negative: if False, clamp all output values to >= 0
+
+    Returns:
+        list[float] of 1440 values
+    """
+    pts = sorted(control_points, key=lambda x: x[0])
+    profile = []
+    idx = 0
+    n = len(pts)
+    for m in range(MINUTES_PER_DAY):
+        while idx + 1 < n and pts[idx + 1][0] <= m:
+            idx += 1
+        t0, v0 = pts[idx]
+        if idx + 1 >= n:
+            val = v0
+        else:
+            t1, v1 = pts[idx + 1]
+            span = t1 - t0
+            mu = (m - t0) / span if span > 0 else 1.0
+            val = v0 + (v1 - v0) * (1.0 - math.cos(mu * math.pi)) / 2.0
+        if not allow_negative:
+            val = max(0.0, val)
+        profile.append(val)
+    return profile
+
+
 def generate_rates_day(rate_type, params, seed):
     """Generate a single-day (1440-minute) per-minute rate profile in p/kWh.
 
     Args:
-        rate_type: One of "single", "dual", "triple", "halfhourly"
+        rate_type: One of "single", "dual", "triple", "halfhourly", "negative_halfhourly"
         params: dict of rate parameters (see per-type docs below)
-        seed: integer seed for halfhourly random generation
+        seed: integer seed for halfhourly/negative_halfhourly random generation
 
     Returns:
         list[float] of 1440 values
@@ -95,15 +130,21 @@ def generate_rates_day(rate_type, params, seed):
             else:
                 profile[m] = shoulder_rate
 
-    elif rate_type == "halfhourly":
-        # 48 half-hourly slots, randomly priced, each slot is 30 minutes
+    elif rate_type in ("halfhourly", "negative_halfhourly"):
+        # Smooth profile: cosine-interpolate between randomly placed peaks and troughs
+        allow_negative = rate_type == "negative_halfhourly"
         rng = random.Random(seed)
-        base = float(params.get("base_rate", 10.0))
-        spread = float(params.get("spread", 20.0))
-        slots = [max(0.0, base + rng.uniform(-spread / 2, spread / 2)) for _ in range(48)]
-        for m in range(MINUTES_PER_DAY):
-            slot = m // 30
-            profile[m] = slots[slot]
+        base = float(params.get("base_rate", 15.0))
+        low = float(params.get("low_rate", 5.0))
+        high = float(params.get("high_rate", 30.0))
+        num_highs = int(params.get("num_highs", 2))
+        num_lows = int(params.get("num_lows", 2))
+        control_points = [(0, base), (MINUTES_PER_DAY - 1, base)]
+        for _ in range(num_highs):
+            control_points.append((rng.randint(0, MINUTES_PER_DAY - 1), high))
+        for _ in range(num_lows):
+            control_points.append((rng.randint(0, MINUTES_PER_DAY - 1), low))
+        profile = _smooth_rate_profile(control_points, allow_negative)
 
     return profile
 
@@ -228,7 +269,7 @@ def generate_random_scenario(scenario_id, seed):
     sunset_minute = rng.randint(PV_SUNSET_MINUTE_MIN, PV_SUNSET_MINUTE_MAX)
 
     # --- Import rate ---
-    import_rate_type = rng.choice(RATE_TYPES)
+    import_rate_type = rng.choice(IMPORT_RATE_TYPES)
     import_rate_params = _sample_rate_params(rng, import_rate_type, cheap=True)
     import_rate_seed = rng.randint(0, 2**31)
 
@@ -1039,7 +1080,19 @@ def _sample_rate_params(rng, rate_type, cheap):
             "peak_end_hhmm": "{}:00".format(peak_end_hour),
         }
 
-    else:  # halfhourly
-        base = round(rng.uniform(10.0, 25.0), 2)
-        spread = round(rng.uniform(10.0, 30.0), 2)
-        return {"base_rate": base, "spread": spread}
+    elif rate_type == "halfhourly":
+        if cheap:
+            base = round(rng.uniform(15.0, 25.0), 2)
+            low = round(rng.uniform(3.0, 12.0), 2)
+            high = round(rng.uniform(30.0, 50.0), 2)
+        else:
+            base = round(rng.uniform(8.0, 18.0), 2)
+            low = round(rng.uniform(2.0, 8.0), 2)
+            high = round(rng.uniform(18.0, 30.0), 2)
+        return {"base_rate": base, "low_rate": low, "high_rate": high, "num_highs": rng.randint(1, 3), "num_lows": rng.randint(1, 3)}
+
+    else:  # negative_halfhourly — troughs go negative to test plunge-pricing scenarios
+        base = round(rng.uniform(5.0, 15.0), 2)
+        low = round(rng.uniform(-15.0, 0.0), 2)
+        high = round(rng.uniform(25.0, 50.0), 2)
+        return {"base_rate": base, "low_rate": low, "high_rate": high, "num_highs": rng.randint(1, 3), "num_lows": rng.randint(1, 3)}

apps/predbat/tests/test_solcast.py

@@ -167,6 +167,7 @@ def __init__(self):
             solcast_poll_hours=4,
             forecast_solar=None,
             forecast_solar_max_age=4,
+            forecast_solar_open_meteo_backup=False,
             pv_forecast_today=None,
             pv_forecast_tomorrow=None,
             pv_forecast_d3=None,
@@ -1712,6 +1713,119 @@ def create_mock_session(*args, **kwargs):
     return failed
 
 
+def test_fetch_pv_forecast_forecast_solar_open_meteo_backup_on_failure(my_predbat):
+    """
+    When forecast.solar returns no data and forecast_solar_open_meteo_backup is True,
+    fetch_pv_forecast falls back to Open-Meteo.
+    """
+    print("  - test_fetch_pv_forecast_forecast_solar_open_meteo_backup_on_failure")
+    failed = False
+
+    test_api = create_test_solar_api()
+    try:
+        test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}]
+        test_api.solar.forecast_solar_open_meteo_backup = True
+        test_api.solar.open_meteo_forecast_max_age = 1.0
+        # forecast.solar returns a server error — download_forecast_solar_data returns ([], 0)
+        test_api.set_mock_response("forecast.solar", {"error": "server error"}, 500)
+        # Open-Meteo returns valid hourly data
+        test_api.set_mock_response(
+            "api.open-meteo.com",
+            {
+                "hourly": {
+                    "time": ["2025-06-15T12:00", "2025-06-15T13:00", "2025-06-15T14:00"],
+                    "global_tilted_irradiance": [500.0, 600.0, 550.0],
+                    "temperature_2m": [25.0, 25.0, 25.0],
+                    "wind_speed_10m": [1.0, 1.0, 1.0],
+                }
+            },
+        )
+        test_api.set_mock_response(
+            "ensemble-api.open-meteo.com",
+            {
+                "hourly": {
+                    "time": ["2025-06-15T12:00", "2025-06-15T13:00", "2025-06-15T14:00"],
+                    "global_tilted_irradiance_member01": [400.0, 480.0, 440.0],
+                }
+            },
+        )
+
+        def create_mock_session(*args, **kwargs):
+            """Create a mock aiohttp session."""
+            return test_api.mock_aiohttp_session()
+
+        with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session):
+            run_async(test_api.solar.fetch_pv_forecast())
+
+        # Open-Meteo should have been called (fallback activated)
+        open_meteo_calls = [r for r in test_api.request_log if "open-meteo.com" in r["url"]]
+        if len(open_meteo_calls) == 0:
+            print("ERROR: Expected Open-Meteo API call during fallback, got none")
+            failed = True
+
+        # Forecast data should have been published (came from Open-Meteo)
+        if f"sensor.{test_api.mock_base.prefix}_pv_today" not in test_api.dashboard_items:
+            print("ERROR: Expected pv_today sensor to be published after Open-Meteo fallback")
+            failed = True
+
+    finally:
+        test_api.cleanup()
+
+    return failed
+
+
+def test_fetch_pv_forecast_forecast_solar_open_meteo_backup_not_used_on_success(my_predbat):
+    """
+    When forecast.solar returns data successfully, Open-Meteo backup is not called
+    even when forecast_solar_open_meteo_backup is True.
+    """
+    print("  - test_fetch_pv_forecast_forecast_solar_open_meteo_backup_not_used_on_success")
+    failed = False
+
+    test_api = create_test_solar_api()
+    try:
+        test_api.solar.forecast_solar = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}]
+        test_api.solar.forecast_solar_open_meteo_backup = True
+        # forecast.solar returns valid data
+        test_api.set_mock_response(
+            "forecast.solar",
+            {
+                "result": {
+                    "watts": {
+                        "2025-06-15T12:00:00+0000": 500,
+                        "2025-06-15T12:30:00+0000": 600,
+                    }
+                },
+                "message": {"info": {"time": "2025-06-15T11:30:00+0000"}},
+            },
+            200,
+        )
+
+        def create_mock_session(*args, **kwargs):
+            """Create a mock aiohttp session."""
+            return test_api.mock_aiohttp_session()
+
+        with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session):
+            run_async(test_api.solar.fetch_pv_forecast())
+
+        # Open-Meteo should NOT have been called
+        open_meteo_calls = [r for r in test_api.request_log if "open-meteo.com" in r["url"]]
+        if len(open_meteo_calls) != 0:
+            print(f"ERROR: Expected no Open-Meteo calls when forecast.solar succeeds, got {len(open_meteo_calls)}")
+            failed = True
+
+        # Forecast.Solar should have been called and data published
+        forecast_calls = [r for r in test_api.request_log if "forecast.solar" in r["url"]]
+        if len(forecast_calls) == 0:
+            print("ERROR: Expected Forecast.Solar API call, got none")
+            failed = True
+
+    finally:
+        test_api.cleanup()
+
+    return failed
+
+
 def test_fetch_pv_forecast_ha_sensors(my_predbat):
     """
     Integration test: fetch_pv_forecast using HA sensors (Solcast integration).
@@ -3270,6 +3384,8 @@ def run_solcast_tests(my_predbat):
     # Integration tests (one per mode)
     failed |= test_fetch_pv_forecast_solcast_direct(my_predbat)
     failed |= test_fetch_pv_forecast_forecast_solar(my_predbat)
+    failed |= test_fetch_pv_forecast_forecast_solar_open_meteo_backup_on_failure(my_predbat)
+    failed |= test_fetch_pv_forecast_forecast_solar_open_meteo_backup_not_used_on_success(my_predbat)
     failed |= test_fetch_pv_forecast_ha_sensors(my_predbat)
 
     # 15-minute resolution tests

docs/apps-yaml.md

@@ -1460,6 +1460,22 @@ Optionally you can set an api_key for personal or professional accounts and you
 
 Note you can omit any of these settings for a default value. They do not have to be exact if you use Predbat auto calibration for PV to improve the data quality.
 
+### Open-Meteo backup for Forecast.solar
+
+If you set `forecast_solar_open_meteo_backup: true`, Predbat will automatically fall back to the [Open-Meteo](#open-meteo-solar-forecast) API whenever Forecast.solar returns no data (for example, due to a server error, rate limiting, or an outage).
+
+When the fallback is active, Predbat derives the Open-Meteo request from the same `forecast_solar` configuration entries (latitude, longitude, postcode, declination, azimuth, kwp, efficiency), so no extra configuration is needed. If you also have an `open_meteo_forecast` section configured, that configuration is used for the backup request instead, which lets you apply Open-Meteo-specific options such as `shading_factors`.
+
+```yaml
+  forecast_solar:
+    - postcode: SW1A 2AB
+      kwp: 3
+      azimuth: 45
+      declination: 45
+      efficiency: 0.95
+  forecast_solar_open_meteo_backup: true
+```
+
 ## Open-Meteo Solar Forecast
 
 [Open-Meteo](https://open-meteo.com/) is a free, open-source weather API that provides solar irradiance forecasts with no API key required.