Merge pull request #24 from dawidlinek/feat/15-min-frequency

Feat/15 min frequency

Author: dawidlinek

data_pipeline.yaml

@@ -0,0 +1,95 @@
+sources:
+- class: EntsoeSource
+  params:
+    country_code: PL
+    api_key: YOUR_API_KEY_HERE
+    type:
+    - load
+    - price
+    prefer_15min: false
+- class: OpenMeteoSource
+  params:
+    latitude: 52.2297
+    longitude: 21.0122
+    horizon: 7
+    model: jma_seamless
+    columns:
+    - temperature_2m
+    - rain
+    - showers
+    - snowfall
+    - relative_humidity_2m
+    - dew_point_2m
+    - apparent_temperature
+    - precipitation
+    - weather_code
+    - surface_pressure
+    - pressure_msl
+    - cloud_cover
+    - wind_speed_10m
+    - wind_direction_10m
+    prefix: warsaw
+    proxy_url: null
+- class: CalendarSource
+  params:
+    country: PL
+    timezone: Europe/Warsaw
+    holidays: binary
+    weekday: onehot
+    hour: false
+    month: false
+    daylight: true
+    prefix: ''
+    freq: 1h
+transformers:
+- class: TimezoneTransformer
+  params:
+    target_tz: Europe/Warsaw
+    daily_columns:
+    - load_forecast_daily_min
+    - load_forecast_daily_max
+- class: ResampleTransformer
+  params:
+    freq: 1h
+    method: linear
+    columns: null
+- class: LagTransformer
+  params:
+    columns:
+    - load_actual
+    - price
+    lags:
+    - 1
+    - 2
+    - 7
+    freq: day
+- class: LagTransformer
+  params:
+    columns:
+    - is_monday
+    - is_tuesday
+    - daylight_hours
+    - is_wednesday
+    - is_thursday
+    - is_friday
+    - is_saturday
+    - is_sunday
+    - load_forecast
+    - is_holiday
+    lags:
+    - -7
+    - -6
+    - -5
+    - -4
+    - -3
+    - -2
+    - -1
+    - 0
+    freq: day
+validators:
+- class: NullCheckValidator
+  params:
+    columns:
+    - load_actual
+    - price
+    allow_nulls: false

docs/src/content/docs/data-sources/entsoe.mdx

@@ -25,6 +25,7 @@ source = EntsoeSource(
     country_code="PL",
     api_key=os.environ.get("ENTSOE_API_KEY"),
     type=["load", "price"],
+    prefer_15min=False,
 )
 
 df = source.fetch(
@@ -40,6 +41,7 @@ df = source.fetch(
 | `country_code` | str | Yes | Country/zone code (see table below) |
 | `api_key` | str | Yes | ENTSOE API key |
 | `type` | List[str] | Yes | Data types: `load`, `price`, `generation` |
+| `prefer_15min` | bool | No | Defaults to `False`. If `True`, prioritizes fetching 15-minute resolution data (particularly relevant for prices). |
 
 ---
 
@@ -58,6 +60,10 @@ df = source.fetch(
 |--------|-------------|
 | `price` | Day-ahead electricity price (EUR/MWh) |
 
+<Aside type="caution">
+  **Germany Price Data:** ENTSOE publishes prices by bidding zone. Since the 2018 split, use the country code `"DE_LU"` (Germany/Luxembourg) to fetch German electricity prices. Using `"DE"` will return an empty list of prices.
+</Aside>
+
 ### Generation Data (`type=["generation"]`)
 
 All supported generation types:

docs/src/content/docs/data-sources/openmeteo.mdx

@@ -42,6 +42,35 @@ df = source.fetch(
 | `model` | str | No | `"jma_seamless"` | Weather model |
 | `columns` | List[str] | No | DEFAULT_COLUMNS | Weather variables |
 | `prefix` | str | No | `""` | Prefix for column names |
+| `temporal_resolution` | str \| None | No | `None` | Time step of returned data — see below |
+
+### Temporal Resolution
+
+| Value | Time step | Notes |
+|-------|-----------|-------|
+| `None` *(default)* | 1 h | Standard hourly data; no extra API parameter sent |
+| `"hourly_3"` | 3 h | 8 data points per day |
+| `"hourly_6"` | 6 h | 4 data points per day |
+| `"native"` | model-dependent | Uses the weather model's native resolution; step is inferred automatically from the response |
+
+```python
+# 3-hourly data — useful for coarser feature sets or faster pipelines
+source = OpenMeteoSource(
+    latitude=52.2297,
+    longitude=21.0122,
+    horizon=7,
+    prefix="warsaw",
+    temporal_resolution="hourly_3",
+)
+
+# Model-native resolution
+source = OpenMeteoSource(
+    latitude=52.2297,
+    longitude=21.0122,
+    model="icon_seamless",
+    temporal_resolution="native",
+)
+```
 
 ## Rate Limiting
 

docs/src/content/docs/model-pipeline/overview.mdx

@@ -32,6 +32,7 @@ report = pipeline.run(
     test_end="2024-03-01",
     target="price",
     horizon=7,
+    freq="1h",                   # Optional. Allows sub-hourly testing like "15min"
     save_dir="results",
 )
 ```
@@ -172,6 +173,7 @@ Models automatically apply `StandardScaler`:
 | `test_end` | str | Required | Test period end |
 | `target` | str | `"price"` | Target column name |
 | `horizon` | int | `7` | Max forecast horizon (days) |
+| `freq` | str | `"1h"` | Forecasting frequency (e.g., `"15min"`) |
 | `save_dir` | str | `None` | Directory for incremental results |
 | `forecast_only` | bool | `False` | Skip evaluators/exporters; `test_start`/`test_end` default to `"today"` if omitted |
 

docs/src/content/docs/transformers/resample.mdx

@@ -7,7 +7,7 @@ import { Aside } from '@astrojs/starlight/components';
 
 # ResampleTransformer
 
-Resamples data to a specified frequency with interpolation to fill gaps.
+Resamples data to a specified frequency, applying aggregation (when downsampling) or interpolation (when upsampling).
 
 ## Basic Usage
 
@@ -23,7 +23,7 @@ df = transformer.transform(df)
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `freq` | str | `"1h"` | Pandas frequency string |
-| `method` | str | `"linear"` | Interpolation method |
+| `method` | str | `"linear"` | Method specifying aggregation and interpolation behavior |
 | `columns` | list[str] \| str \| None | `None` | Columns to interpolate. If None, all columns are interpolated. |
 
 ## Frequency Strings
@@ -35,17 +35,24 @@ df = transformer.transform(df)
 | `"15min"` | 15 minutes |
 | `"1D"` | 1 day |
 
-## Interpolation Methods
+## Methods
 
-| Method | Description |
-|--------|-------------|
-| `"linear"` | Linear interpolation between points |
-| `"ffill"` | Forward fill (use last known value) |
-| `"bfill"` | Backward fill (use next known value) |
+The `method` parameter controls behavior for **both** directions:
+- **Downsampling** (e.g. 15 min → 1 h) – selects the aggregation function.
+- **Upsampling** (e.g. 1 h → 15 min) – selects how NaN gaps are filled.
+
+| Method | Aggregation (↓) | Fill Method (↑) |
+|--------|----------------|-----------------|
+| `"linear"` | `mean` | Linear interpolation |
+| `"ffill"` | `first` | Forward-fill |
+| `"bfill"` | `last` | Backward-fill |
+| `"sum"` | `sum` | Forward-fill |
+| `"first"` | `first` | Forward-fill |
+| `"last"` | `last` | Backward-fill |
 
 ## Selecting Columns
 
-By default, interpolation is applied to all columns. You can restrict interpolation to specific columns using the `columns` parameter. Other columns will be resampled (introducing NaNs) but not interpolated.
+By default, the resampling method is applied to all columns. You can restrict it to specific columns using the `columns` parameter. Other columns will be resampled (introducing NaNs/dropping points) but not processed by the specified method.
 
 ```python
 # Interpolate only 'price', leave 'load' with NaNs where gaps occur

docs/src/content/docs/workflow/overview.mdx

@@ -57,6 +57,7 @@ model_pipeline:
   test_end: "2024-07-01"
   target: price
   horizon: 7
+  freq: 1h
 ```
 
 **Forecast-only variant** — add `forecast_only: true` and use date keywords so the file runs daily without editing:
@@ -72,6 +73,7 @@ model_pipeline:
   path: model_pipeline.yaml
   target: price
   horizon: 7
+  freq: 1h                 # set to 15min for 15-minute forecasting
   forecast_only: true      # test_start/test_end default to "today"; skips evaluators and exporters
 ```
 
@@ -94,6 +96,7 @@ model_pipeline:
 | `data_cache` | bool \| str | `False` | Data caching (True, False, or path) |
 | `model_target` | str | `"price"` | Target column name |
 | `model_horizon` | int | `7` | Max forecast horizon (days) |
+| `model_freq` | str | `"1h"` | Forecasting frequency (e.g., `"15min"`) |
 | `max_processes` | int | `None` | Worker process count |
 | `threads_per_process` | int | `None` | Threads per process |
 | `cache_path` | str | `None` | Cache directory (overrides `data_cache` if set) |

epftoolbox2/data/sources/calendar.py

@@ -45,6 +45,7 @@ def __init__(
         month: Union[str, bool] = False,
         daylight: bool = False,
         prefix: str = "",
+        freq: str = "1h",
     ):
         self.country = country.upper()
         self._validate_country()
@@ -60,6 +61,7 @@ def __init__(
         self.month = month
         self.daylight = daylight
         self.prefix = prefix
+        self.freq = freq
 
         self._validate_config()
 
@@ -86,7 +88,7 @@ def fetch(self, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame:
         start = start.tz_convert("UTC") if start.tzinfo else start.tz_localize("UTC")
         end = end.tz_convert("UTC") if end.tzinfo else end.tz_localize("UTC")
 
-        index = pd.date_range(start, end, freq="1h", tz="UTC")
+        index = pd.date_range(start, end, freq=self.freq, tz="UTC")
         local_index = index.tz_convert(self.timezone)
 
         df = pd.DataFrame(index=index)
@@ -132,7 +134,11 @@ def _add_weekday(self, df: pd.DataFrame, local_index: pd.DatetimeIndex) -> pd.Da
         return df
 
     def _add_hour(self, df: pd.DataFrame, local_index: pd.DatetimeIndex) -> pd.DataFrame:
-        hour_series = pd.Series(local_index.hour, index=df.index)
+        if self.freq == "15min":
+            values = local_index.hour * 4 + local_index.minute // 15
+        else:
+            values = local_index.hour
+        hour_series = pd.Series(values, index=df.index)
 
         if self.hour == "number":
             df[f"{self.prefix}hour"] = hour_series

epftoolbox2/data/sources/entsoe.py

@@ -176,10 +176,17 @@ class EntsoeSource(DataSource):
 
     API_URL = "https://web-api.tp.entsoe.eu/api"
 
-    def __init__(self, country_code: str, api_key: str, type: List[str]):
+    def __init__(
+        self,
+        country_code: str,
+        api_key: str,
+        type: List[str],
+        prefer_15min: bool = False,
+    ):
         self.country_code = country_code
         self.api_key = api_key
         self.type = type
+        self.prefer_15min = prefer_15min
         self._area_name, self._area_code = lookup_area(country_code)
         self.session = requests.Session()
 
@@ -384,12 +391,15 @@ def _fetch_price(self, start: pd.Timestamp, end: pd.Timestamp) -> pd.DataFrame:
         price_dict = self._parse_prices(xml)
 
         series = pd.Series()
-        if price_dict["15min"] is not None and len(price_dict["15min"]) > 0:
-            series = price_dict["15min"]
-        elif price_dict["30min"] is not None and len(price_dict["30min"]) > 0:
-            series = price_dict["30min"]
-        elif price_dict["60min"] is not None and len(price_dict["60min"]) > 0:
-            series = price_dict["60min"]
+        order = (
+            ("15min", "30min", "60min")
+            if self.prefer_15min
+            else ("60min", "30min", "15min")
+        )
+        for res in order:
+            if price_dict.get(res) is not None and len(price_dict[res]) > 0:
+                series = price_dict[res]
+                break
 
         series = series.truncate(before=start, after=end)