Merge pull request #503 from paulhomes/weather

Cache Open-Meteo JSON reponse locally to reduce the number of API calls and add resilience

Author: davidusb-geek

.gitignore

@@ -47,6 +47,8 @@ MANIFEST
 
 # Local session data 
 data/actionLogs.txt
+data/debug-*.csv
+data/cached-open-meteo-forecast.json
 data/entities/*.json
 
 # PyInstaller

docs/forecasts.md

@@ -94,6 +94,18 @@ curl -i -H 'Content-Type:application/json' -X POST -d {} http://localhost:5000/a
 curl -i -H 'Content-Type:application/json' -X POST -d '{"weather_forecast_cache_only":true}' http://localhost:5000/action/naive-mpc-optim
 ```
 
+#### Caching Open-Meteo Weather Service Usage
+
+When you have EMHASS configured to use the Open-Meteo weather service, to minimize API calls to the service, and to provide
+resilience in case of transient connectivity issues, EMHASS caches successful calls to the Open-Meteo API in a
+`cached-open-meteo-forecast.json` file in the data directory. The JSON file contains the default 3 days of weather forecast data.
+This Open-Meteo cache is independent of the PV cache discussed above and will be used even when the PV cache is not enabled.
+By default, when the JSON file is older than 30 minutes, attempts will be made to replace it with a more recent version
+from the Open-Meteo weather service. It will only be replaced if this is successful. If any errors occur the older version
+will continue to be used until a new version can been fetched. The maximum cache age, with a default value of 30 minutes, can be
+configured using the `open_meteo_cache_max_age` setting in config.json or as a parameter in EMHASS REST API calls.
+The value is specified in minutes. If you want to disable caching you can specify a value of 0.
+
 #### Adjusting PV Forecasts using machine learning
 EMHASS provides methods to adjust the PV power forecast using machine learning regression techniques. The adjustment process consists of two steps: training a regression model using historical PV data and then applying the trained model to correct new PV forecasts.
 

src/emhass/data/associations.csv

@@ -46,6 +46,7 @@ optim_conf,battery_dynamic_min,battery_dynamic_min
 optim_conf,weight_battery_discharge,weight_battery_discharge
 optim_conf,weight_battery_charge,weight_battery_charge
 optim_conf,weather_forecast_method,weather_forecast_method
+optim_conf,open_meteo_cache_max_age,open_meteo_cache_max_age
 optim_conf,def_start_timestep,start_timesteps_of_each_deferrable_load,list_start_timesteps_of_each_deferrable_load
 optim_conf,def_end_timestep,end_timesteps_of_each_deferrable_load,list_end_timesteps_of_each_deferrable_load
 optim_conf,list_hp_periods,load_peak_hour_periods

src/emhass/data/config_defaults.json

@@ -43,6 +43,7 @@
     0
   ],
   "weather_forecast_method": "open-meteo",
+  "open_meteo_cache_max_age": 30,
   "load_forecast_method": "naive",
   "delta_forecast_daily": 1,
   "load_cost_forecast_method": "hp_hc_periods",

src/emhass/forecast.py

@@ -23,6 +23,7 @@
 from pvlib.pvsystem import PVSystem
 from pvlib.temperature import TEMPERATURE_MODEL_PARAMETERS
 from requests import get
+from requests.exceptions import RequestException
 
 from sklearn.pipeline import make_pipeline
 from sklearn.preprocessing import StandardScaler
@@ -215,6 +216,92 @@ def __init__(
                         0 : self.params["passed_data"]["prediction_horizon"]
                     ]
 
+    def get_cached_open_meteo_forecast_json(
+        self,
+        max_age: Optional[int] = 30,
+    ) -> dict:
+        r"""
+        Get weather forecast json from Open-Meteo and cache it for re-use.
+        The response json is cached in the local file system and returned
+        on subsequent calls until it is older than max_age, at which point
+        attempts will be made to replace it with a new version.
+        The cached version will not be overwritten until a new version has
+        been successfully fetched from Open-Meteo.
+        In the event of connectivity issues, the cached version will continue
+        to be returned until such time as a new version can be successfully
+        fetched from Open-Meteo.
+        If you want to force reload, pass max_age value of zero.
+
+        :param max_age: The maximum age of the cached json file, in minutes,
+            before it is discarded and a new version fetched from Open-Meteo.
+            Defaults to 30 minutes.
+        :type max_age: int, optional
+        :return: The json containing the Open-Meteo forecast data
+        :rtype: dict
+
+        """
+        json_path = os.path.abspath(
+            self.emhass_conf["data_path"] / "cached-open-meteo-forecast.json"
+        )
+        # The cached JSON file is always loaded, if it exists, as it is also a fallback
+        # in case the REST API call to Open-Meteo fails - the cached JSON will continue to
+        # be used until it can successfully fetch a new version from Open-Meteo.
+        data = None
+        use_cache = False
+        if os.path.exists(json_path):
+            delta = datetime.now() - datetime.fromtimestamp(os.path.getmtime(json_path))
+            json_age = int(delta / timedelta(seconds=60))
+            use_cache = json_age < max_age
+            self.logger.info("Loading existing cached Open-Meteo JSON file: %s", json_path)
+            with open(json_path) as json_file:
+                data = json.load(json_file)
+            if use_cache:
+                self.logger.info("The cached Open-Meteo JSON file is recent (age=%.0fm, max_age=%sm)",
+                                 json_age, max_age)
+            else:
+                self.logger.info("The cached Open-Meteo JSON file is old (age=%.0fm, max_age=%sm)",
+                                 json_age, max_age)
+
+        if not use_cache:
+            self.logger.info("Fetching a new weather forecast from Open-Meteo")
+            headers = {"User-Agent": "EMHASS", "Accept": "application/json"}
+            url = (
+                "https://api.open-meteo.com/v1/forecast?"
+                + "latitude="
+                + str(round(self.lat, 2))
+                + "&longitude="
+                + str(round(self.lon, 2))
+                + "&minutely_15="
+                + "temperature_2m,"
+                + "relative_humidity_2m,"
+                + "rain,"
+                + "cloud_cover,"
+                + "wind_speed_10m,"
+                + "shortwave_radiation_instant,"
+                + "diffuse_radiation_instant,"
+                + "direct_normal_irradiance_instant"
+                + "&timezone="
+                + quote(str(self.time_zone), safe="")
+            )
+            try:
+                response = get(url, headers=headers)
+                self.logger.debug("Returned HTTP status code: %s", response.status_code)
+                response.raise_for_status()
+                """import bz2 # Uncomment to save a serialized data for tests
+                import _pickle as cPickle
+                with bz2.BZ2File("data/test_response_openmeteo_get_method.pbz2", "w") as f:
+                    cPickle.dump(response, f)"""
+                data = response.json()
+                self.logger.info("Saving response in Open-Meteo JSON cache file: %s", json_path)
+                with open(json_path, "w") as json_file:
+                    json.dump(response.json(), json_file, indent=2)
+            except RequestException:
+                self.logger.error("Failed to fetch weather forecast from Open-Meteo", exc_info=True)
+                if data is not None:
+                    self.logger.warning("Returning old cached data until next Open-Meteo attempt")
+
+        return data
+
     def get_weather_forecast(
         self,
         method: Optional[str] = "open-meteo",
@@ -248,32 +335,7 @@ def get_weather_forecast(
             method == "open-meteo" or method == "scrapper"
         ):  # The scrapper option is being left here for backward compatibility
             if not os.path.isfile(w_forecast_cache_path):
-                headers = {"User-Agent": "EMHASS", "Accept": "application/json"}
-                url = (
-                    "https://api.open-meteo.com/v1/forecast?"
-                    + "latitude="
-                    + str(round(self.lat, 2))
-                    + "&longitude="
-                    + str(round(self.lon, 2))
-                    + "&minutely_15="
-                    + "temperature_2m,"
-                    + "relative_humidity_2m,"
-                    + "rain,"
-                    + "cloud_cover,"
-                    + "wind_speed_10m,"
-                    + "shortwave_radiation_instant,"
-                    + "diffuse_radiation_instant,"
-                    + "direct_normal_irradiance_instant"
-                    + "&timezone="
-                    + quote(str(self.time_zone), safe="")
-                )
-                response = get(url, headers=headers)
-                """import bz2 # Uncomment to save a serialized data for tests
-                import _pickle as cPickle
-                with bz2.BZ2File("data/test_response_openmeteo_get_method.pbz2", "w") as f: 
-                    cPickle.dump(response, f)"""
-
-                data_raw = response.json()
+                data_raw = self.get_cached_open_meteo_forecast_json(self.optim_conf["open_meteo_cache_max_age"])
                 data_15min = pd.DataFrame.from_dict(data_raw["minutely_15"])
                 data_15min["time"] = pd.to_datetime(data_15min["time"])
                 data_15min.set_index("time", inplace=True)

src/emhass/static/data/param_definitions.json

@@ -150,6 +150,12 @@
       ],
       "default_value": "open-meteo"
     },
+    "open_meteo_cache_max_age": {
+      "friendly_name": "Open-Meteo Cache Max Age",
+      "Description": "The maximum age, in minutes, of the cached open-meteo json response, after which a new version will be fetched from Open-Meteo. Defaults to 30.",
+      "input": "int",
+      "default_value": 30
+    },
     "maximum_power_from_grid": {
       "friendly_name": "Max power from grid",
       "Description": "The maximum power that can be supplied by the utility grid in Watts (consumption). Defaults to 9000.",