Merge pull request #23 from dawidlinek/fix--daylight-changes-errors

Fix  daylight changes errors

Author: dawidlinek

epftoolbox2/data/sources/entsoe.py

@@ -241,16 +241,19 @@ def fetch(self, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame:
                 chunk_data = self._fetch_chunk(chunk_start, chunk_end)
 
                 for dtype in self.type:
-                    if dtype in chunk_data:
-                        all_results[dtype].append(chunk_data[dtype])
+                    chunk_df = chunk_data.get(dtype)
+                    if chunk_df is not None and not chunk_df.empty:
+                        all_results[dtype].append(chunk_df)
                 progress.advance(task)
 
         dataframes = []
         for dtype in self.type:
             if all_results[dtype]:
                 df = pd.concat(all_results[dtype]).sort_index()
-                df = df[~df.index.duplicated(keep="first")]
-                dataframes.append(df)
+                if df.index.has_duplicates:
+                    df = df.groupby(level=0).first()
+                if not df.empty:
+                    dataframes.append(df)
 
         elapsed = time.time() - start_time
         self._log_success(elapsed)
@@ -301,7 +304,9 @@ def _fetch_load(self, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame:
             start,
             end,
         )
-        dfs.append(self._parse_loads(xml, "A16"))
+        df_actual = self._parse_loads(xml, "A16")
+        if not df_actual.empty:
+            dfs.append(df_actual)
 
         xml = self._api_request(
             {
@@ -312,7 +317,9 @@ def _fetch_load(self, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame:
             start,
             end,
         )
-        dfs.append(self._parse_loads(xml, "A01"))
+        df_forecast = self._parse_loads(xml, "A01")
+        if not df_forecast.empty:
+            dfs.append(df_forecast)
 
         xml = self._api_request(
             {
@@ -323,10 +330,17 @@ def _fetch_load(self, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame:
             start,
             end,
         )
-        dfs.append(self._parse_loads(xml, "A31"))
+        df_week = self._parse_loads(xml, "A31")
+        if not df_week.empty:
+            dfs.append(df_week)
+        elif xml is not None:
+            self.logger.warning(f"ENTSOE [{self._area_name}]: A31 (week-ahead forecast) data available but no min/max values found")
+
+        if not dfs:
+            return pd.DataFrame()
 
         result = pd.concat(dfs, axis=1)
-        return result.truncate(before=start, after=end)
+        return result.truncate(before=start - pd.Timedelta(days=1), after=end)
 
     def _fetch_generation(self, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame:
         dfs = []
@@ -342,17 +356,6 @@ def _fetch_generation(self, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFr
         )
         dfs.append(self._parse_generation(xml).add_prefix("generation_"))
 
-        xml = self._api_request(
-            {
-                "documentType": "A69",
-                "processType": "A01",
-                "in_Domain": self._area_code,
-            },
-            start,
-            end,
-        )
-        dfs.append(self._parse_generation(xml).add_prefix("generation_"))
-
         xml = self._api_request(
             {
                 "documentType": "A69",
@@ -381,10 +384,12 @@ def _fetch_price(self, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame:
         price_dict = self._parse_prices(xml)
 
         series = pd.Series()
-        if price_dict["60min"] is not None and len(price_dict["60min"]) > 0:
-            series = price_dict["60min"]
-        elif price_dict["15min"] is not None and len(price_dict["15min"]) > 0:
+        if price_dict["15min"] is not None and len(price_dict["15min"]) > 0:
             series = price_dict["15min"]
+        elif price_dict["30min"] is not None and len(price_dict["30min"]) > 0:
+            series = price_dict["30min"]
+        elif price_dict["60min"] is not None and len(price_dict["60min"]) > 0:
+            series = price_dict["60min"]
 
         series = series.truncate(before=start, after=end)
 
@@ -433,15 +438,15 @@ def _parse_loads(self, xml_text: str, process_type: str) -> pd.DataFrame:
                 elif bsn_type and bsn_type.text == "A61":
                     series_max_list.append(t)
 
-            series_min = pd.concat(series_min_list) if series_min_list else pd.Series(dtype=float)
-            series_max = pd.concat(series_max_list) if series_max_list else pd.Series(dtype=float)
+            result_dict = {}
+            if series_min_list:
+                series_min = pd.concat(series_min_list).sort_index()
+                result_dict["load_forecast_daily_min"] = series_min
+            if series_max_list:
+                series_max = pd.concat(series_max_list).sort_index()
+                result_dict["load_forecast_daily_max"] = series_max
 
-            return pd.DataFrame(
-                {
-                    "load_forecast_daily_min": series_min,
-                    "load_forecast_daily_max": series_max,
-                }
-            )
+            return pd.DataFrame(result_dict) if result_dict else pd.DataFrame()
 
     def _parse_generation(self, xml_text: str) -> pd.DataFrame:
         if xml_text is None:

epftoolbox2/data/sources/open_meteo.py

@@ -304,7 +304,4 @@ def get_cache_config(self) -> dict:
             "prefix": self.prefix,
         }
 
-        if self.proxy_url:
-            config["proxy_url"] = self.proxy_url
-
         return config

epftoolbox2/data/transformers/timezone.py

@@ -1,21 +1,36 @@
+import numpy as np
 import pandas as pd
 from zoneinfo import ZoneInfo
 from .base import Transformer
 
 
 class TimezoneTransformer(Transformer):
     """Transformer that converts DataFrame index timezone.
+
+    Optionally snaps sparse daily columns to local midnight after conversion.
+    When daily values (e.g. load_forecast_daily_min) are stamped at UTC 00:00,
+    tz conversion shifts them to 01:00/02:00 local time. A subsequent ffill
+    resample would then cover 00:00 of that local day with the *previous*
+    day's value. Passing such columns via ``daily_columns`` moves each
+    non-null value back to 00:00 of the same local calendar day.
+
     Example:
-        >>> transformer = TimezoneTransformer(target_tz="Europe/Warsaw")
+        >>> transformer = TimezoneTransformer(
+        ...     target_tz="Europe/Warsaw",
+        ...     daily_columns=["load_forecast_daily_min", "load_forecast_daily_max"],
+        ... )
         >>> df = transformer.transform(df)
     """
 
-    def __init__(self, target_tz: str):
+    def __init__(self, target_tz: str, daily_columns: list[str] = ["load_forecast_daily_min", "load_forecast_daily_max"]):
         """
         Args:
             target_tz: Target timezone name (e.g., "Europe/Warsaw", "America/New_York")
+            daily_columns: Columns holding one value per day that should be
+                re-snapped to local midnight after tz conversion.
         """
         self.target_tz = target_tz
+        self.daily_columns = daily_columns
         self._validate_timezone()
 
     def _validate_timezone(self) -> None:
@@ -35,4 +50,26 @@ def transform(self, df: pd.DataFrame) -> pd.DataFrame:
         else:
             result.index = result.index.tz_convert(self.target_tz)
 
+        # Snap daily columns to local midnight after tz conversion
+        for col in self.daily_columns:
+            if col not in result.columns:
+                continue
+
+            non_null = result[col].dropna()
+            if non_null.empty:
+                continue
+
+            midnight_index = non_null.index.normalize()
+            needs_shift = non_null.index != midnight_index
+            if not needs_shift.any():
+                continue
+
+            to_move = non_null[needs_shift]
+            target_idx = midnight_index[needs_shift]
+            valid = target_idx.isin(result.index)
+
+            result.loc[to_move.index[valid], col] = np.nan
+            new_series = pd.Series(to_move.values[valid], index=target_idx[valid])
+            result.loc[new_series.index, col] = new_series.values
+
         return result

epftoolbox2/models/base.py

@@ -250,6 +250,12 @@ def _precompute_expanded_predictors(
 
     def _expand_predictors(self, horizon: int, hour: int = 0) -> List[str]:
         result = []
+        if callable(self.predictors):
+            try:
+                n_params = len(inspect.signature(self.predictors).parameters)
+            except (ValueError, TypeError):
+                n_params = 1
+            self.predictors = self.predictors(horizon, hour) if n_params >= 2 else self.predictors(horizon)
         for col in self.predictors:
             if callable(col):
                 try: