first pass

.devcontainer/devcontainer.json

@@ -1,6 +1,7 @@
 // See https://aka.ms/vscode-remote/devcontainer.json for format details.
 {
-	"image": "ludeeus/container:integration-debian",
+	//"image": "ludeeus/container:integration-debian",
+	"image": "ghcr.io/ludeeus/devcontainer/integration:stable",
 	"name": "Nordpool integration development",
 	"context": "..",
 	"appPort": [
@@ -16,7 +17,7 @@
 	"settings": {
 		"files.eol": "\n",
 		"editor.tabSize": 4,
-		"terminal.integrated.shell.linux": "/bin/bash",
+		//"terminal.integrated.shell.linux": "/bin/bash",
 		"python.pythonPath": "/usr/bin/python3",
 		"python.analysis.autoSearchPaths": false,
 		"python.linting.pylintEnabled": true,

custom_components/nordpool/__init__.py

@@ -3,27 +3,58 @@
 from datetime import datetime, timedelta
 from functools import partial
 from random import randint
+from typing import Union
 
+import aiohttp
+import backoff
 import voluptuous as vol
 from homeassistant.config_entries import ConfigEntry
 from homeassistant.core import Config, HomeAssistant
 from homeassistant.helpers.aiohttp_client import async_get_clientsession
 from homeassistant.helpers.dispatcher import async_dispatcher_send
-from homeassistant.helpers.event import (async_call_later,
-                                         async_track_time_change)
+from homeassistant.helpers.event import async_call_later, async_track_time_change
 from homeassistant.util import dt as dt_utils
 from pytz import timezone
 
 from .aio_price import AioPrices
 from .events import async_track_time_change_in_tz
+from .misc import test_valid_nordpooldata, stock, predicate
 
 DOMAIN = "nordpool"
 _LOGGER = logging.getLogger(__name__)
 RANDOM_MINUTE = randint(0, 10)
 RANDOM_SECOND = randint(0, 59)
-EVENT_NEW_DATA = "nordpool_update"
+EVENT_NEW_DATA = "nordpool_update_new_data"
 _CURRENCY_LIST = ["DKK", "EUR", "NOK", "SEK"]
 
+_REGIONS = {
+    "DK1": ["DKK", "Denmark", 0.25],
+    "DK2": ["DKK", "Denmark", 0.25],
+    "FI": ["EUR", "Finland", 0.24],
+    "EE": ["EUR", "Estonia", 0.20],
+    "LT": ["EUR", "Lithuania", 0.21],
+    "LV": ["EUR", "Latvia", 0.21],
+    "Oslo": ["NOK", "Norway", 0.25],
+    "Kr.sand": ["NOK", "Norway", 0.25],
+    "Bergen": ["NOK", "Norway", 0.25],
+    "Molde": ["NOK", "Norway", 0.25],
+    "Tr.heim": ["NOK", "Norway", 0.25],
+    "Tromsø": ["NOK", "Norway", 0.25],
+    "SE1": ["SEK", "Sweden", 0.25],
+    "SE2": ["SEK", "Sweden", 0.25],
+    "SE3": ["SEK", "Sweden", 0.25],
+    "SE4": ["SEK", "Sweden", 0.25],
+    # What zone is this?
+    "SYS": ["EUR", "System zone", 0.25],
+    "FR": ["EUR", "France", 0.055],
+    "NL": ["EUR", "Netherlands", 0.21],
+    "BE": ["EUR", "Belgium", 0.21],
+    "AT": ["EUR", "Austria", 0.20],
+    # Tax is disabled for now, i need to split the areas
+    # to handle the tax.
+    "DE-LU": ["EUR", "Germany and Luxembourg", 0],
+}
+
 
 CONFIG_SCHEMA = vol.Schema({DOMAIN: vol.Schema({})}, extra=vol.ALLOW_EXTRA)
 
@@ -44,45 +75,82 @@
 
 
 class NordpoolData:
+    """Handles the updates."""
+
     def __init__(self, hass: HomeAssistant):
         self._hass = hass
         self._last_tick = None
         self._data = defaultdict(dict)
-        self._tomorrow_valid = False
         self.currency = []
         self.listeners = []
 
-    async def _update(self, type_="today", dt=None):
+    async def _update(self, *args, type_="today", dt=None) -> bool:
         _LOGGER.debug("calling _update %s %s", type_, dt)
         hass = self._hass
         client = async_get_clientsession(hass)
 
         if dt is None:
             dt = dt_utils.now()
 
-        # We dont really need today and morrow
-        # when the region is in another timezone
-        # as we request data for 3 days anyway.
-        # Keeping this for now, but this should be changed.
-        for currency in self.currency:
+        @backoff.on_predicate(backoff.expo, predicate=predicate, logger=_LOGGER)
+        @backoff.on_exception(backoff.expo, aiohttp.ClientError, logger=_LOGGER)
+        async def really_update(currency: str, end_date: datetime) -> Union[bool, None]:
+            """Requests the data from nordpool and retries on http errors or missing/wrong data."""
+            func_now = dt_utils.now()
             spot = AioPrices(currency, client)
-            data = await spot.hourly(end_date=dt)
-            if data:
-                self._data[currency][type_] = data["areas"]
+            data = await spot.hourly(end_date=end_date)
+            # We only verify the the areas that has the correct currency, example AT is always inf for all other currency then EUR
+            # Now this will fail for any users that has a non local currency for the region they selected.
+            # Thats a problem for another day..
+            regions_to_verify = [k for k, v in _REGIONS.items() if v[0] == currency]
+            data_ok = test_valid_nordpooldata(data, region=regions_to_verify)
+
+            if data_ok is False:
+                np_should_have_released_new_data = stock(func_now).replace(
+                    hour=13, minute=RANDOM_MINUTE, second=RANDOM_SECOND
+                )
+
+                if type_ == "tomorrow":
+                    if stock(func_now) >= np_should_have_released_new_data:
+                        _LOGGER.info(
+                            "New data should be available, it does not exist or isnt valid so we will retry the request later"
+                        )
+                        return False
+
+                    else:
+                        _LOGGER.info("No new data is available")
+                        # Need to handle the None
+                        # Give up, a new request will pulling the data 1300ish
+                        return None
+                else:
+                    return False
+
             else:
-                _LOGGER.info("Some crap happend, retrying request later.")
-                async_call_later(hass, 20, partial(self._update, type_=type_, dt=dt))
+                self._data[currency][type_] = data["areas"]
 
-    async def update_today(self, n: datetime):
-        _LOGGER.debug("Updating tomorrows prices.")
-        await self._update("today")
+            return True
 
-    async def update_tomorrow(self, n: datetime):
+        attempts = []
+        for currency in self.currency:
+            update_attempt = await really_update(currency, dt)
+            attempts.append(update_attempt)
+
+        return all(attempts)
+
+    async def update_today(self, n: datetime) -> bool:
+        """Gets the prices for today"""
+        _LOGGER.debug("Updating todays prices.")
+        return await self._update("today")
+
+    async def update_tomorrow(self, n: datetime) -> bool:
+        """Get the prices for tomorrow"""
         _LOGGER.debug("Updating tomorrows prices.")
-        await self._update(type_="tomorrow", dt=dt_utils.now() + timedelta(hours=24))
-        self._tomorrow_valid = True
+        result = await self._update(
+            type_="tomorrow", dt=dt_utils.now() + timedelta(hours=24)
+        )
+        return result
 
-    async def _someday(self, area: str, currency: str, day: str):
+    async def _someday(self, area: str, currency: str, day: str) -> Union[dict, None]:
         """Returns todays or tomorrows prices in a area in the currency"""
         if currency not in _CURRENCY_LIST:
             raise ValueError(
@@ -91,23 +159,20 @@ async def _someday(self, area: str, currency: str, day: str):
             )
 
         # This is needed as the currency is
-        # set in the sensor.
+        # set in the sensor and we need to pull for the first time.
         if currency not in self.currency:
             self.currency.append(currency)
             await self.update_today(None)
             await self.update_tomorrow(None)
 
         return self._data.get(currency, {}).get(day, {}).get(area)
 
-    def tomorrow_valid(self) -> bool:
-        return self._tomorrow_valid
-
-    async def today(self, area: str, currency: str) -> dict:
+    async def today(self, area: str, currency: str) -> Union[dict, None]:
         """Returns todays prices in a area in the requested currency"""
         res = await self._someday(area, currency, "today")
         return res
 
-    async def tomorrow(self, area: str, currency: str):
+    async def tomorrow(self, area: str, currency: str) -> Union[dict, None]:
         """Returns tomorrows prices in a area in the requested currency"""
         res = await self._someday(area, currency, "tomorrow")
         return res
@@ -120,26 +185,39 @@ async def _dry_setup(hass: HomeAssistant, config: Config) -> bool:
         api = NordpoolData(hass)
         hass.data[DOMAIN] = api
 
-        async def new_day_cb(n):
+        async def new_day_cb(n) -> None:
             """Cb to handle some house keeping when it a new day."""
             _LOGGER.debug("Called new_day_cb callback")
-            api._tomorrow_valid = False
 
             for curr in api.currency:
-                if not len(api._data[curr]["tomorrow"]):
-                    api._data[curr]["today"] = await api.update_today(None)
+                tom = api._data[curr]["tomorrow"]
+                regions_to_verify = [k for k, v in _REGIONS.items() if v[0] == curr]
+                _LOGGER.debug("Checking that data we already have for tomorrow is ok.")
+                data_ok = test_valid_nordpooldata(tom, region=regions_to_verify)
+                if data_ok:
+                    api._data[curr]["today"] = tom
                 else:
-                    api._data[curr]["today"] = api._data[curr]["tomorrow"]
+                    await api.update_today(None)
+
                 api._data[curr]["tomorrow"] = {}
+                _LOGGER.debug(
+                    "Clear tomorrow for %s, %s", curr, api._data[curr]["tomorrow"]
+                )
 
             async_dispatcher_send(hass, EVENT_NEW_DATA)
 
-        async def new_hr(n):
+        async def new_hr(n: datetime) -> None:
             """Callback to tell the sensors to update on a new hour."""
             _LOGGER.debug("Called new_hr callback")
+
+            # We don't want to notify the sensor about this hour as this is
+            # handled by the new_day_cb anyway.
+            if n.hour == 0:
+                return
+
             async_dispatcher_send(hass, EVENT_NEW_DATA)
 
-        async def new_data_cb(n):
+        async def new_data_cb(n: datetime) -> None:
             """Callback to fetch new data for tomorrows prices at 1300ish CET
             and notify any sensors, about the new data
             """
@@ -182,7 +260,7 @@ async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
         hass.config_entries.async_forward_entry_setup(entry, "sensor")
     )
 
-    # entry.add_update_listener(async_reload_entry)
+    entry.add_update_listener(async_reload_entry)
     return res
 
 
@@ -191,9 +269,10 @@ async def async_unload_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool:
     unload_ok = await hass.config_entries.async_forward_entry_unload(entry, "sensor")
 
     if unload_ok:
-        for unsub in hass.data[DOMAIN].listeners:
-            unsub()
-        hass.data.pop(DOMAIN)
+        if DOMAIN in hass.data:
+            for unsub in hass.data[DOMAIN].listeners:
+                unsub()
+            hass.data.pop(DOMAIN)
 
         return True
 

custom_components/nordpool/aio_price.py

@@ -140,21 +140,22 @@ def join_result_for_correct_time(results, dt):
 
 
 class AioPrices(Prices):
+    """Aioprices"""
+
     def __init__(self, currency, client, tz=None):
         super().__init__(currency)
         self.client = client
         self.tz = tz
         self.API_URL_CURRENCY = "https://www.nordpoolgroup.com/api/marketdata/page/%s"
 
     async def _io(self, url, **kwargs):
-
         resp = await self.client.get(url, params=kwargs)
         _LOGGER.debug("requested %s %s", resp.url, kwargs)
 
         return await resp.json()
 
     async def _fetch_json(self, data_type, end_date=None, areas=None):
-        """ Fetch JSON from API """
+        """Fetch JSON from API"""
         # If end_date isn't set, default to tomorrow
         if end_date is None:
             end_date = date.today() + timedelta(days=1)
@@ -168,7 +169,8 @@ async def _fetch_json(self, data_type, end_date=None, areas=None):
             endDate=end_date.strftime("%d-%m-%Y"),
         )
 
-    async def fetch(self, data_type, end_date=None, areas=[]):
+    # https://github.com/custom-components/integration_blueprint/issues/71
+    async def fetch(self, data_type, end_date=None, areas=None):
         """
         Fetch data from API.
         Inputs:
@@ -189,12 +191,9 @@ async def fetch(self, data_type, end_date=None, areas=[]):
                 - list of values (dictionary with start and endtime and value)
                 - possible other values, such as min, max, average for hourly
         """
+        if areas is None:
+            areas = []
 
-        # Check how to handle all time zone in this,
-        # dunno how to do this yet.
-        # stock = datetime.utcnow().astimezone(tz.gettz("Europe/Stockholm"))
-        # stock_offset = stock.utcoffset().total_seconds()
-        # compare utc offset
         if self.tz == tz.gettz("Europe/Stockholm"):
             data = await self._fetch_json(data_type, end_date, areas)
             return self._parse_json(data, areas)
@@ -248,30 +247,48 @@ async def fetch(self, data_type, end_date=None, areas=[]):
             res = await asyncio.gather(*jobs)
 
             raw = [self._parse_json(i, areas) for i in res]
-            return join_result_for_correct_time(raw, end_date)
+            result = join_result_for_correct_time(raw, end_date)
+            # test_result = test_valid_nordpooldata(result)
+            # _LOGGER.debug("DATA STATUS %s", test_result)
+            return result
+
+    async def hourly(self, end_date=None, areas=None):
+        """Helper to fetch hourly data, see Prices.fetch()"""
+        if areas is None:
+            areas = []
 
-    async def hourly(self, end_date=None, areas=[]):
-        """ Helper to fetch hourly data, see Prices.fetch() """
         return await self.fetch(self.HOURLY, end_date, areas)
 
-    async def daily(self, end_date=None, areas=[]):
-        """ Helper to fetch daily data, see Prices.fetch() """
+    async def daily(self, end_date=None, areas=None):
+        """Helper to fetch daily data, see Prices.fetch()"""
+        if areas is None:
+            areas = []
+
         return await self.fetch(self.DAILY, end_date, areas)
 
-    async def weekly(self, end_date=None, areas=[]):
-        """ Helper to fetch weekly data, see Prices.fetch() """
+    async def weekly(self, end_date=None, areas=None):
+        """Helper to fetch weekly data, see Prices.fetch()"""
+        if areas is None:
+            areas = []
+
         return await self.fetch(self.WEEKLY, end_date, areas)
 
-    async def monthly(self, end_date=None, areas=[]):
-        """ Helper to fetch monthly data, see Prices.fetch() """
+    async def monthly(self, end_date=None, areas=None):
+        """Helper to fetch monthly data, see Prices.fetch()"""
+        if areas is None:
+            areas = []
+
         return await self.fetch(self.MONTHLY, end_date, areas)
 
-    async def yearly(self, end_date=None, areas=[]):
-        """ Helper to fetch yearly data, see Prices.fetch() """
+    async def yearly(self, end_date=None, areas=None):
+        """Helper to fetch yearly data, see Prices.fetch()"""
+        if areas is None:
+            areas = []
+
         return await self.fetch(self.YEARLY, end_date, areas)
 
     def _conv_to_float(self, s):
-        """ Convert numbers to float. Return infinity, if conversion fails. """
+        """Convert numbers to float. Return infinity, if conversion fails."""
         try:
             return float(s.replace(",", ".").replace(" ", ""))
         except ValueError: