feat: Integrate missing PECD data (#71)

daniel-rdt · tgilon · web-flow · commit 6ab3be3704eb · 2025-08-11T14:03:19.000+02:00
* feat: add configuration for pecd fill_gaps_method for missing data

* feat: add SolarPV for utility and rooftop with differentiation

* fix: adjust config name of solar-pv carrier

* fix: make LFSolarPV import more generic to differentiation of more nodes

* feat: add Exception if no input data for specified PECD tech

* feat: add onshore wind to tyndp renewable carriers and pecd profiles

* fix: check in add existing renwables in case non of the specified carriers are included

* fix: add back onwind and solar to renewable carriers until tyndp carriers are added to the model

* feat: Slightly improve value error message

* feat: add todo for redundant renewable carriers

* feat: custom fix for skiprows and typo

* feat: add collect rules for clean_pecd_datas and build_renewable_profiles_pecds

* feat: add logger warning for missing files

* feat: introduce safe_pyear helper function

* feat: make handling of GB and UK more consistent

* feat: adjust wording of logger about existing capacities

* doc: document compatibility with Solar CSP

* doc: document new config option for available pyears in all configs and documentation files

* doc: add release note

* feat: add verbose option to safe_pyear helper function

* Apply suggestions from code review

Co-authored-by: Thomas Gilon &lt;thomas.gilon@openenergytransition.org&gt;

* fix: remove manual 2050 fallback to 2040 for PECD data

* feat: adjust input and output types in safe_pyear

* Apply suggestions from code review

Co-authored-by: Thomas Gilon &lt;thomas.gilon@openenergytransition.org&gt;

* feat: refactor input_data_pecd function for easier readability

* apply suggestions from code review

* doc: add documentation for retrieve_tyndp_pecd_data rule

---------

Co-authored-by: Thomas Gilon &lt;thomas.gilon@openenergytransition.org&gt;
diff --git a/config/config.default.yaml b/config/config.default.yaml
@@ -145,6 +145,11 @@ electricity:
 
   pecd_renewable_profiles:
     enable: false
+    fill_gaps_method: zero
+    available_years:
+    - 2030
+    - 2040
+    - 2050
     technologies:
       Wind_Offshore:
       - offwind-ac-fb-r
@@ -393,6 +398,10 @@ load:
     enable: true
     interpolate_limit: 3
     time_shift_for_large_gaps: 1w
+  available_years_tyndp:
+  - 2030
+  - 2040
+  - 2050
   manual_adjustments: true
   scaling_factor: 1.0
   fixed_year: false
diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml
@@ -37,11 +37,18 @@ co2_budget:
 electricity:
   base_network: tyndp
 
-  renewable_carriers: [solar, solar-hsat, onwind, hydro]
-  tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
+  # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
+  renewable_carriers: [onwind, solar, solar-hsat, hydro]
+  tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
 
   pecd_renewable_profiles:
     enable: true
+    fill_gaps_method: zero
+    # Complete PECD data is only available for the years 2030, 2040
+    # TODO: adjust once udpated 2050 PECD data is available
+    available_years:
+    - 2030
+    - 2040
     technologies:
       Wind_Offshore:
       - offwind-ac-fb-r
@@ -52,6 +59,12 @@ electricity:
       - offwind-dc-fl-oh
       - offwind-h2-fb-oh
       - offwind-h2-fl-oh
+      LFSolarPVRooftop:
+      - solar-pv-rooftop
+      LFSolarPVUtility:
+      - solar-pv-utility
+      Wind_Onshore:
+      - onwind
 
   estimate_renewable_capacities:
     # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation
@@ -76,6 +89,10 @@ load:
   source: tyndp # opsd, tyndp
   fill_gaps:
     enable: false
+  available_years_tyndp:
+  - 2030
+  - 2040
+  - 2050
   manual_adjustments: false
   supplement_synthetic: false
 
diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml
@@ -49,11 +49,18 @@ electricity:
     Store: [H2]
     Link: [H2 pipeline]
 
-  renewable_carriers: [solar, solar-hsat, onwind]
-  tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
+  # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
+  renewable_carriers: [onwind, solar, solar-hsat]
+  tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
 
   pecd_renewable_profiles:
     enable: true
+    fill_gaps_method: zero
+    # Complete PECD data is only available for the years 2030, 2040
+    # TODO: adjust once udpated 2050 PECD data is available
+    available_years:
+    - 2030
+    - 2040
     technologies:
       Wind_Offshore:
       - offwind-ac-fb-r
@@ -64,6 +71,12 @@ electricity:
       - offwind-dc-fl-oh
       - offwind-h2-fb-oh
       - offwind-h2-fl-oh
+      LFSolarPVRooftop:
+      - solar-pv-rooftop
+      LFSolarPVUtility:
+      - solar-pv-utility
+      Wind_Onshore:
+      - onwind
 
   estimate_renewable_capacities:
     # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation
@@ -103,6 +116,10 @@ load:
   source: tyndp # opsd, tyndp
   fill_gaps:
     enable: false
+  available_years_tyndp:
+  - 2030
+  - 2040
+  - 2050
   manual_adjustments: false
   supplement_synthetic: false
 
diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv
@@ -32,7 +32,9 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model.
 tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv-rooftop, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh}",List of TYNDP renewable generators to include in the model. Technologies covered by specified TYNDP renewable carriers need to be removed from `estimate_renewable_carriers` technology list.
 pecd_renewable_profiles,,,
--- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}.
+-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {LFSolarPVUtility, LFSolarPVRooftop, Wind_Offshore, Wind_Onshore, CSP_noStorage, CSP_withStorage_7h_dispatched, CSP_withStorage_7h_preDispatch}.
+-- fill_gaps_method,,str,The chosen method for filling gaps in the PECD data to the modelled nodes. Can be either `zero` to fill with zero values or any other aggregation method such as `mean`, `median`, `max` or similar.
+-- available_years,--,list,"List of years for which PECD data is available."
 -- technologies,,,
 -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``.
 -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles.
diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv
@@ -4,6 +4,7 @@ fill_gaps,--,--,"Gaps filling strategy used."
 -- enable,bool,"{true, false}","Whether to fill gaps using interpolation for small gaps and time shift for large gaps."
 -- interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly."
 -- time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings."
+available_years_tyndp,--,list,"List of years for which TYNDP demand data is available."
 manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`."
 scaling_factor,--,float,"Global correction factor for the load time series."
 fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year"
diff --git a/doc/data-retrieval.rst b/doc/data-retrieval.rst
@@ -258,3 +258,11 @@ Data in this section is retrieved and extracted in rules specified in ``rules/re
 - **Link:** https://cdn.jsdelivr.net/gh/gavinr/world-countries-centroids@v1.0.0/dist/countries.geojson
 - **License:** MIT License
 - **Description:** Contains centroids of world countries in GeoJson.
+
+``data/tyndp_2024_bundle/PECD``
+
+- **Source:** TYNDP 2024 Scenarios, https://2024.entsos-tyndp-scenarios.eu/download/
+- **Link:** https://storage.googleapis.com/open-tyndp-data-store/PECD.zip
+- **License:** CC-BY 4.0
+- **Description:** Contains hourly PECD v3.1 capacity factor time series for Solar PV rooftop,
+  Solar PV utility, Onshore Wind, Offshore Wind and Solar CSP.
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
@@ -12,7 +12,7 @@ Release Notes
 Upcoming Open-TYNDP Release
 ================
 
-
+* Add complete compatibility for processing and preparation of PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP) (https://github.com/open-energy-transition/open-tyndp/pull/71). These profiles are used for the TYNDP 2024 and replace the default ERA5- and SARAH3-based profiles processed with Atlite. This implementation serves to facilitate a sub-workflow for creation of the renewable profiles, but does not yet attach them to any technologies.
 
 
 Upcoming PyPSA-Eur Release
diff --git a/doc/retrieve.rst b/doc/retrieve.rst
@@ -168,3 +168,8 @@ None.
 **Outputs**
 
 - ``data/countries_centroids.geojson``
+
+Rule ``retrieve_tyndp_pecd_data``
+====================================
+
+.. automodule:: retrieve_tyndp_pecd_data
diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk
@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: MIT
 
+from scripts._helpers import safe_pyear
+
 
 def input_elec_demand(w):
     return {
@@ -398,6 +400,12 @@ rule clean_pecd_data:
     params:
         snapshots=config_provider("snapshots"),
         drop_leap_day=config_provider("enable", "drop_leap_day"),
+        fill_gaps_method=config_provider(
+            "electricity", "pecd_renewable_profiles", "fill_gaps_method"
+        ),
+        available_years=config_provider(
+            "electricity", "pecd_renewable_profiles", "available_years"
+        ),
     input:
         offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx",
         onshore_buses=resources("busmap_base_s_all.csv"),
@@ -418,19 +426,26 @@ rule clean_pecd_data:
 
 
 def input_data_pecd(w):
+    available_years = config_provider(
+        "electricity", "pecd_renewable_profiles", "available_years"
+    )(w)
+    planning_horizons = config_provider("scenario", "planning_horizons")(w)
+    safe_pyears = set(
+        safe_pyear(year, available_years, "PECD", verbose=False)
+        for year in planning_horizons
+    )
     return {
         f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv")
-        for pyear in set(
-            config_provider("scenario", "planning_horizons")(w)
-        ).intersection([2030, 2040])
-        # Complete PECD data is only available for the years 2030, 2040
-        # TODO: adjust if udpated 2050 data available
+        for pyear in safe_pyears
     }
 
 
 rule build_renewable_profiles_pecd:
     params:
         planning_horizons=config_provider("scenario", "planning_horizons"),
+        available_years=config_provider(
+            "electricity", "pecd_renewable_profiles", "available_years"
+        ),
     input:
         unpack(input_data_pecd),
     output:
@@ -1032,6 +1047,7 @@ if config["load"]["source"] == "tyndp":
             planning_horizons=config_provider("scenario", "planning_horizons"),
             snapshots=config_provider("snapshots"),
             scenario=config_provider("tyndp_scenario"),
+            available_years=config_provider("load", "available_years_tyndp"),
         input:
             electricity_demand="data/tyndp_2024_bundle/Demand Profiles",
         output:
diff --git a/rules/collect.smk b/rules/collect.smk
@@ -90,3 +90,27 @@ rule plot_power_networks_clustered:
             **config["scenario"],
             run=config["run"]["name"],
         ),
+
+
+rule clean_pecd_datas:
+    input:
+        lambda w: expand(
+            resources("pecd_data_{technology}_{planning_horizons}.csv"),
+            **config["scenario"],
+            run=config["run"]["name"],
+            technology=config_provider(
+                "electricity", "pecd_renewable_profiles", "technologies"
+            )(w),
+        ),
+
+
+rule build_renewable_profiles_pecds:
+    input:
+        lambda w: expand(
+            resources("profile_pecd_{clusters}_{technology}.nc"),
+            **config["scenario"],
+            run=config["run"]["name"],
+            technology=config_provider(
+                "electricity", "pecd_renewable_profiles", "technologies"
+            )(w),
+        ),
diff --git a/scripts/_helpers.py b/scripts/_helpers.py
@@ -10,6 +10,7 @@
 import os
 import re
 import time
+from bisect import bisect_right
 from functools import partial, wraps
 from pathlib import Path
 from tempfile import NamedTemporaryFile
@@ -1130,3 +1131,49 @@ def extract_grid_data_tyndp(
     links.index = links.apply(make_index, axis=1, prefix=carrier)
 
     return links
+
+
+def safe_pyear(
+    year: int | str,
+    available_years: list = [2030, 2040, 2050],
+    source: str = "TYNDP",
+    verbose: bool = True,
+):
+    """
+    Checks and adjusts whether a given pyear is in the available years of a given data source. If not, it falls back to the previous available year.
+
+    Parameters
+    ----------
+    year : int
+        Planning horizon year which will be checked and possibly adjusted to previous available year.
+    available_years : list, optional
+        List of available years. Defaults to [2030, 2040, 2050].
+    source : str, optional
+        Source of the data for which availability will be checked. For logging purpose only. Defaults to "TYNDP".
+    verbose : bool, optional
+        Whether to activate verbose logging. Defaults to True.
+
+    Returns
+    -------
+    year_new : int
+        Safe pyear adjusted for available years
+    """
+
+    if not available_years:
+        raise ValueError(
+            "No `available_years` provided. Expected a non-empty list of years."
+        )
+    if not isinstance(year, int):
+        year = int(year)
+    if year not in available_years:
+        year_new = available_years[
+            bisect_right(sorted(available_years), year, lo=1) - 1
+        ]
+        if verbose:
+            logger.warning(
+                f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}."
+            )
+    else:
+        year_new = year
+
+    return year_new
diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py
@@ -103,6 +103,12 @@ def add_existing_renewables(
 
     irena = irena.unstack().reset_index()
 
+    if not set(tech_map).intersection(renewable_carriers):
+        # TODO: adjust once TYNDP existing capacities are added.
+        logger.info(
+            "No existing capacities to add for specified renewable carriers. Existing capacities of TYNDP renewable carriers will be added separately."
+        )
+
     for carrier, tech in tech_map.items():
         if carrier not in renewable_carriers:
             continue
@@ -146,7 +152,9 @@ def add_existing_renewables(
                     df_agg.at[name, "bus"] = bus
                     df_agg.at[name, "resource_class"] = bin_id
 
-    df_agg["resource_class"] = df_agg["resource_class"].fillna(0)
+    df_agg["resource_class"] = (
+        df_agg["resource_class"].fillna(0) if "resource_class" in df_agg.columns else 0
+    )
 
 
 def add_power_capacities_installed_before_baseyear(
diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py
@@ -22,12 +22,12 @@
 
 import logging
 
-import numpy as np
 import pandas as pd
 import xarray as xr
 
 from scripts._helpers import (
     configure_logging,
+    safe_pyear,
     set_scenario_config,
 )
 
@@ -55,16 +55,10 @@
             f"Extract PECD capacity factor time series for year {year} for technology {technology}..."
         )
         year_i = year
-        if int(year) not in [2030, 2040, 2050]:
-            year = np.clip(10 * (year // 10), 2030, 2050)
-            logger.warning(
-                f"TYNDP PECD data unavailable for planning horizon. Falling back to previous available year {year}."
-            )
-        if year == 2050:
-            logger.warning(
-                "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data."
-            )
-            year = 2040
+        # falling back to latest available pyear if not in list of available years
+        year = safe_pyear(
+            year, available_years=snakemake.params.available_years, source="PECD"
+        )
 
         profile = (
             pd.read_csv(
diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py
diff --git a/scripts/clean_tyndp_demand.py b/scripts/clean_tyndp_demand.py
diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py
