Merge pull request #22 from dawidlinek/feat/extended-evaluation

Feat/extended evaluation

Author: dawidlinek

docs/src/content/docs/evaluators/mae.md

@@ -45,6 +45,8 @@ print(report.summary())
 # 0    OLS  26.0199
 ```
 
-## Creating Custom Evaluators
+## See Also
 
-See [Extending](/epftoolbox2/reference/extending/) for how to create custom evaluators like RMSE or MAPE.
+- [RMSEEvaluator](/epftoolbox2/evaluators/rmse/) — Root Mean Squared Error
+- [rMAEEvaluator](/epftoolbox2/evaluators/rmae/) — Relative MAE (benchmark comparison)
+- [Extending](/epftoolbox2/reference/extending/) — Creating custom evaluators

docs/src/content/docs/evaluators/rmae.md

@@ -0,0 +1,65 @@
+---
+title: rMAEEvaluator
+description: Relative Mean Absolute Error metric
+---
+
+# rMAEEvaluator
+
+Calculates relative Mean Absolute Error (rMAE) — the ratio of a model's MAE to a base model's MAE. This is a standard metric in the electricity price forecasting literature (Lago et al., 2021) for benchmarking model performance.
+
+## Formula
+
+rMAE = MAE(model) / MAE(base_model)
+
+Where both MAEs are computed on the same data slice (same time period, same grouping).
+
+**Interpretation:**
+- rMAE < 1 — model **outperforms** the base model
+- rMAE = 1 — model performs **equally** to the base model
+- rMAE > 1 — model **underperforms** the base model
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `base_model` | str | Required | Name of the model to use as the benchmark |
+
+## Basic Usage
+
+```python
+from epftoolbox2.evaluators import rMAEEvaluator
+
+evaluator = rMAEEvaluator(base_model="OLS")
+```
+
+## In Pipeline
+
+```python
+from epftoolbox2.pipelines import ModelPipeline
+from epftoolbox2.models import OLSModel, LassoCVModel
+from epftoolbox2.evaluators import MAEEvaluator, rMAEEvaluator
+from epftoolbox2.exporters import TerminalExporter
+
+pipeline = (
+    ModelPipeline()
+    .add_model(OLSModel(predictors=predictors, name="OLS"))
+    .add_model(LassoCVModel(predictors=predictors, cv=7, name="LassoCV"))
+    .add_evaluator(MAEEvaluator())
+    .add_evaluator(rMAEEvaluator(base_model="OLS"))
+    .add_exporter(TerminalExporter())
+)
+
+report = pipeline.run(...)
+print(report.summary())
+#      model       MAE   rMAE
+# 0      OLS  26.0199  1.0000
+# 1  LassoCV  24.8100  0.9535
+```
+
+The rMAE is computed per data slice, so grouped views (`by_hour`, `by_horizon`, etc.) each show the relative performance for that specific group.
+
+## Notes
+
+- The `base_model` name must match the `name` parameter of one of the models in the pipeline.
+- If the base model's MAE is zero for a given group, rMAE returns `inf`.
+- rMAE is serializable to YAML and works with `ModelPipeline.save()`/`load()`.

docs/src/content/docs/evaluators/rmse.md

@@ -0,0 +1,47 @@
+---
+title: RMSEEvaluator
+description: Root Mean Squared Error metric
+---
+
+# RMSEEvaluator
+
+Calculates Root Mean Squared Error (RMSE) between predictions and actual values. RMSE penalizes larger errors more heavily than MAE.
+
+## Formula
+
+RMSE = √((1/n) × Σ(yᵢ - ŷᵢ)²)
+
+Where:
+- yᵢ = actual value
+- ŷᵢ = predicted value
+- n = number of observations
+
+## Basic Usage
+
+```python
+from epftoolbox2.evaluators import RMSEEvaluator
+
+evaluator = RMSEEvaluator()
+```
+
+## In Pipeline
+
+```python
+from epftoolbox2.pipelines import ModelPipeline
+from epftoolbox2.models import OLSModel
+from epftoolbox2.evaluators import MAEEvaluator, RMSEEvaluator
+from epftoolbox2.exporters import TerminalExporter
+
+pipeline = (
+    ModelPipeline()
+    .add_model(OLSModel(predictors=predictors, name="OLS"))
+    .add_evaluator(MAEEvaluator())
+    .add_evaluator(RMSEEvaluator())
+    .add_exporter(TerminalExporter())
+)
+
+report = pipeline.run(...)
+print(report.summary())
+#    model       MAE     RMSE
+# 0    OLS  26.0199  32.4512
+```

docs/src/content/docs/exporters/csv.md

@@ -0,0 +1,87 @@
+---
+title: CsvExporter
+description: Export results to a wide-format CSV
+---
+
+# CsvExporter
+
+Exports detailed prediction results to a wide-format CSV file. Each row represents a single forecast point, with per-model prediction and error columns.
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `path` | str | Required | Output CSV file path |
+| `extra_columns` | List[str] | `[]` | Columns from the source dataset to include |
+
+## Basic Usage
+
+```python
+from epftoolbox2.exporters import CsvExporter
+
+exporter = CsvExporter("results.csv")
+```
+
+## Output Format
+
+The CSV contains the following columns:
+
+**Base columns:**
+- `run_date` — date the forecast was made
+- `target_date` — date being forecasted
+- `hour` — hour of day (0-23)
+- `horizon` — forecast horizon (1 to max)
+- `day_in_test` — day index in the test period
+- `actual` — actual observed value
+
+**Per-model columns:**
+- `{model}_prediction` — model's prediction
+- `{model}_error` — residual (prediction - actual)
+
+**Extra columns** (optional):
+- Any columns from the source dataset, joined by `target_date` + `hour`
+
+### Example Output
+
+For a pipeline with models `OLS` and `LassoCV`, and `extra_columns=["is_holiday"]`:
+
+| run_date | target_date | hour | horizon | actual | OLS_prediction | OLS_error | LassoCV_prediction | LassoCV_error | is_holiday |
+|----------|-------------|------|---------|--------|---------------|-----------|-------------------|---------------|------------|
+| 2024-02-01 | 2024-02-02 | 0 | 1 | 48.50 | 45.23 | -3.27 | 46.10 | -2.40 | 0 |
+
+## Extra Columns
+
+Use `extra_columns` to include columns from the source dataset (e.g., calendar features, weather data). Columns are joined by matching `target_date` and `hour` from the results to the source dataset's DatetimeIndex.
+
+```python
+exporter = CsvExporter(
+    "results.csv",
+    extra_columns=["is_holiday", "load_forecast", "warsaw_temperature_2m"],
+)
+```
+
+If a requested column does not exist in the source dataset, it is silently skipped.
+
+## In Pipeline
+
+```python
+from epftoolbox2.pipelines import ModelPipeline
+from epftoolbox2.models import OLSModel, LassoCVModel
+from epftoolbox2.evaluators import MAEEvaluator, RMSEEvaluator
+from epftoolbox2.exporters import CsvExporter
+
+pipeline = (
+    ModelPipeline()
+    .add_model(OLSModel(predictors=predictors, name="OLS"))
+    .add_model(LassoCVModel(predictors=predictors, cv=7, name="LassoCV"))
+    .add_evaluator(MAEEvaluator())
+    .add_evaluator(RMSEEvaluator())
+    .add_exporter(CsvExporter(
+        "results.csv",
+        extra_columns=["is_holiday", "load_forecast"],
+    ))
+)
+
+report = pipeline.run(data=df, test_start="2024-02-01", test_end="2024-03-01", target="price", horizon=7)
+# Results saved to results.csv
+```

docs/src/content/docs/reference/api.md

@@ -200,9 +200,19 @@ class LassoCVModel(BaseModel):
 ## Evaluators
 
 ```python
+class Evaluator(ABC):
+    name: str
+    def compute(self, df: pd.DataFrame, **kwargs) -> float: ...
+
 class MAEEvaluator(Evaluator):
     name = "MAE"
-    def compute(self, df: pd.DataFrame) -> float: ...
+
+class RMSEEvaluator(Evaluator):
+    name = "RMSE"
+
+class rMAEEvaluator(Evaluator):
+    name = "rMAE"
+    def __init__(self, base_model: str): ...  # Name of the benchmark model
 ```
 
 ---
@@ -217,4 +227,8 @@ class TerminalExporter(Exporter):
 class ExcelExporter(Exporter):
     def __init__(self, path: str, sheets: List[str] = None): ...
     def export(self, report: EvaluationReport) -> None: ...
+
+class CsvExporter(Exporter):
+    def __init__(self, path: str, extra_columns: List[str] = None): ...
+    def export(self, report: EvaluationReport) -> None: ...
 ```

docs/src/content/docs/reference/extending.md

@@ -130,42 +130,44 @@ from epftoolbox2.evaluators.base import Evaluator
 import pandas as pd
 import numpy as np
 
-class RMSEEvaluator(Evaluator):
-    name = "RMSE"
-    
-    def compute(self, df: pd.DataFrame) -> float:
-        return np.sqrt(((df["prediction"] - df["actual"]) ** 2).mean())
-
-
 class MAPEEvaluator(Evaluator):
     name = "MAPE"
 
-    def compute(self, df: pd.DataFrame) -> float:
+    def compute(self, df: pd.DataFrame, **kwargs) -> float:
         return ((df["prediction"] - df["actual"]).abs() / df["actual"].abs()).mean() * 100
 
 
 class sMAPEEvaluator(Evaluator):
     name = "sMAPE"
 
-    def compute(self, df: pd.DataFrame) -> float:
+    def compute(self, df: pd.DataFrame, **kwargs) -> float:
         numerator = (df["prediction"] - df["actual"]).abs()
         denominator = (df["prediction"].abs() + df["actual"].abs()) / 2
         return (numerator / denominator).mean() * 100
 ```
 
+### Evaluator Interface
+
+Every evaluator must implement:
+
+- **`name`** — class attribute used as the column header in reports
+- **`compute(self, df, **kwargs) -> float`** — aggregate metric over a DataFrame with `prediction` and `actual` columns
+
+The `**kwargs` may include `model_dfs` — a dict of all model DataFrames for the current data slice, used by cross-model evaluators like `rMAEEvaluator`.
+
 ### Using Custom Evaluators
 
 ```python
 from epftoolbox2.pipelines import ModelPipeline
 from epftoolbox2.models import OLSModel
-from epftoolbox2.evaluators import MAEEvaluator
+from epftoolbox2.evaluators import MAEEvaluator, RMSEEvaluator
 from epftoolbox2.exporters import TerminalExporter
 
 pipeline = (
     ModelPipeline()
     .add_model(OLSModel(predictors=predictors, name="OLS"))
     .add_evaluator(MAEEvaluator())
-    .add_evaluator(RMSEEvaluator())   # Custom
+    .add_evaluator(RMSEEvaluator())
     .add_evaluator(MAPEEvaluator())   # Custom
     .add_exporter(TerminalExporter())
 )

epftoolbox2/evaluators/__init__.py

@@ -1,4 +1,6 @@
 from .base import Evaluator
 from .mae import MAEEvaluator
+from .rmae import rMAEEvaluator
+from .rmse import RMSEEvaluator
 
-__all__ = ["Evaluator", "MAEEvaluator"]
+__all__ = ["Evaluator", "MAEEvaluator", "RMSEEvaluator", "rMAEEvaluator"]

epftoolbox2/evaluators/base.py

@@ -6,5 +6,5 @@ class Evaluator(ABC):
     name: str
 
     @abstractmethod
-    def compute(self, df: pd.DataFrame) -> float:
+    def compute(self, df: pd.DataFrame, **kwargs) -> float:
         pass

epftoolbox2/evaluators/mae.py

@@ -5,5 +5,5 @@
 class MAEEvaluator(Evaluator):
     name = "MAE"
 
-    def compute(self, df: pd.DataFrame) -> float:
+    def compute(self, df: pd.DataFrame, **kwargs) -> float:
         return (df["prediction"] - df["actual"]).abs().mean()

epftoolbox2/evaluators/rmae.py

@@ -0,0 +1,29 @@
+import math
+from typing import Dict
+
+import pandas as pd
+from .base import Evaluator
+
+
+class rMAEEvaluator(Evaluator):
+    name = "rMAE"
+
+    def __init__(self, base_model: str):
+        self.base_model = base_model
+
+    def compute(self, df: pd.DataFrame, **kwargs) -> float:
+        model_dfs: Dict[str, pd.DataFrame] = kwargs.get("model_dfs", {})
+        if not model_dfs:
+            raise ValueError(
+                f"rMAE base model '{self.base_model}' not found in pipeline models."
+            )
+        if self.base_model not in model_dfs:
+            return math.nan
+        base_df = model_dfs[self.base_model]
+        if base_df.empty:
+            return math.nan
+        base_mae = (base_df["prediction"] - base_df["actual"]).abs().mean()
+        if base_mae == 0:
+            return float("inf")
+        model_mae = (df["prediction"] - df["actual"]).abs().mean()
+        return model_mae / base_mae