feat: Integrate missing PECD data

config/config.default.yaml

@@ -144,6 +144,11 @@ electricity:
 
   pecd_renewable_profiles:
     enable: false
+    fill_gaps_method: zero
+    available_years:
+    - 2030
+    - 2040
+    - 2050
     technologies:
       Wind_Offshore:
       - offwind-ac-fb-r
@@ -392,6 +397,10 @@ load:
     enable: true
     interpolate_limit: 3
     time_shift_for_large_gaps: 1w
+  available_years_tyndp:
+  - 2030
+  - 2040
+  - 2050
   manual_adjustments: true
   scaling_factor: 1.0
   fixed_year: false

config/config.tyndp.yaml

@@ -37,11 +37,18 @@ co2_budget:
 electricity:
   base_network: tyndp
 
-  renewable_carriers: [solar, solar-hsat, onwind, hydro]
-  tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
+  # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
+  renewable_carriers: [onwind, solar, solar-hsat, hydro]
+  tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
 
   pecd_renewable_profiles:
     enable: true
+    fill_gaps_method: zero
+    # Complete PECD data is only available for the years 2030, 2040
+    # TODO: adjust once udpated 2050 PECD data is available
+    available_years:
+    - 2030
+    - 2040
     technologies:
       Wind_Offshore:
       - offwind-ac-fb-r
@@ -52,6 +59,12 @@ electricity:
       - offwind-dc-fl-oh
       - offwind-h2-fb-oh
       - offwind-h2-fl-oh
+      LFSolarPVRooftop:
+      - solar-pv-rooftop
+      LFSolarPVUtility:
+      - solar-pv-utility
+      Wind_Onshore:
+      - onwind
 
   estimate_renewable_capacities:
     # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation
@@ -76,6 +89,10 @@ load:
   source: tyndp # opsd, tyndp
   fill_gaps:
     enable: false
+  available_years_tyndp:
+  - 2030
+  - 2040
+  - 2050
   manual_adjustments: false
   supplement_synthetic: false
 

config/test/config.tyndp.yaml

@@ -48,11 +48,18 @@ electricity:
     Store: [H2]
     Link: [H2 pipeline]
 
-  renewable_carriers: [solar, solar-hsat, onwind]
-  tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
+  # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
+  renewable_carriers: [onwind, solar, solar-hsat]
+  tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
 
   pecd_renewable_profiles:
     enable: true
+    fill_gaps_method: zero
+    # Complete PECD data is only available for the years 2030, 2040
+    # TODO: adjust once udpated 2050 PECD data is available
+    available_years:
+    - 2030
+    - 2040
     technologies:
       Wind_Offshore:
       - offwind-ac-fb-r
@@ -63,6 +70,12 @@ electricity:
       - offwind-dc-fl-oh
       - offwind-h2-fb-oh
       - offwind-h2-fl-oh
+      LFSolarPVRooftop:
+      - solar-pv-rooftop
+      LFSolarPVUtility:
+      - solar-pv-utility
+      Wind_Onshore:
+      - onwind
 
   estimate_renewable_capacities:
     # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation
@@ -102,6 +115,10 @@ load:
   source: tyndp # opsd, tyndp
   fill_gaps:
     enable: false
+  available_years_tyndp:
+  - 2030
+  - 2040
+  - 2050
   manual_adjustments: false
   supplement_synthetic: false
 

doc/configtables/electricity.csv

@@ -32,7 +32,9 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model.
 tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv-rooftop, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh}",List of TYNDP renewable generators to include in the model. Technologies covered by specified TYNDP renewable carriers need to be removed from `estimate_renewable_carriers` technology list.
 pecd_renewable_profiles,,,
--- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}.
+-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {LFSolarPVUtility, LFSolarPVRooftop, Wind_Offshore, Wind_Onshore, CSP_noStorage, CSP_withStorage_7h_dispatched, CSP_withStorage_7h_preDispatch}.
+-- fill_gaps_method,,str,The chosen method for filling gaps in the PECD data to the modelled nodes. Can be either `zero` to fill with zero values or any other aggregation method such as `mean`, `median`, `max` or similar.
+-- available_years,--,list,"List of years for which PECD data is available."
 -- technologies,,,
 -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``.
 -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles.

doc/configtables/load.csv

@@ -4,6 +4,7 @@ fill_gaps,--,--,"Gaps filling strategy used."
 -- enable,bool,"{true, false}","Whether to fill gaps using interpolation for small gaps and time shift for large gaps."
 -- interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly."
 -- time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings."
+available_years_tyndp,--,list,"List of years for which TYNDP demand data is available."
 manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`."
 scaling_factor,--,float,"Global correction factor for the load time series."
 fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year"

doc/release_notes.rst

@@ -11,7 +11,7 @@ Release Notes
 Upcoming Open-TYNDP Release
 ================
 
-
+* Add complete compatibility for processing and preparation of PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP) (https://github.com/open-energy-transition/open-tyndp/pull/71). These profiles are used for the TYNDP 2024 and replace the default ERA5- and SARAH3-based profiles processed with Atlite. This implementation serves to facilitate a sub-workflow for creation of the renewable profiles, but does not yet attach them to any technologies.
 
 
 Upcoming PyPSA-Eur Release

rules/build_electricity.smk

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: MIT
 
+from scripts._helpers import safe_pyear
+
 
 def input_elec_demand(w):
     return {
@@ -397,6 +399,12 @@ rule clean_pecd_data:
     params:
         snapshots=config_provider("snapshots"),
         drop_leap_day=config_provider("enable", "drop_leap_day"),
+        fill_gaps_method=config_provider(
+            "electricity", "pecd_renewable_profiles", "fill_gaps_method"
+        ),
+        available_years=config_provider(
+            "electricity", "pecd_renewable_profiles", "available_years"
+        ),
     input:
         offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx",
         onshore_buses=resources("busmap_base_s_all.csv"),
@@ -420,16 +428,27 @@ def input_data_pecd(w):
     return {
         f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv")
         for pyear in set(
-            config_provider("scenario", "planning_horizons")(w)
-        ).intersection([2030, 2040])
-        # Complete PECD data is only available for the years 2030, 2040
-        # TODO: adjust if udpated 2050 data available
+            [
+                safe_pyear(
+                    year,
+                    config_provider(
+                        "electricity", "pecd_renewable_profiles", "available_years"
+                    )(w),
+                    "PECD",
+                    verbose=False,
+                )
+                for year in config_provider("scenario", "planning_horizons")(w)
+            ]
+        )
     }
 
 
 rule build_renewable_profiles_pecd:
     params:
         planning_horizons=config_provider("scenario", "planning_horizons"),
+        available_years=config_provider(
+            "electricity", "pecd_renewable_profiles", "available_years"
+        ),
     input:
         unpack(input_data_pecd),
     output:
@@ -1031,6 +1050,7 @@ if config["load"]["source"] == "tyndp":
             planning_horizons=config_provider("scenario", "planning_horizons"),
             snapshots=config_provider("snapshots"),
             scenario=config_provider("tyndp_scenario"),
+            available_years=config_provider("load", "available_years_tyndp"),
         input:
             electricity_demand=directory("data/tyndp_2024_bundle/Demand Profiles"),
         output:

rules/collect.smk

@@ -90,3 +90,27 @@ rule plot_power_networks_clustered:
             **config["scenario"],
             run=config["run"]["name"],
         ),
+
+
+rule clean_pecd_datas:
+    input:
+        lambda w: expand(
+            resources("pecd_data_{technology}_{planning_horizons}.csv"),
+            **config["scenario"],
+            run=config["run"]["name"],
+            technology=config_provider(
+                "electricity", "pecd_renewable_profiles", "technologies"
+            )(w),
+        ),
+
+
+rule build_renewable_profiles_pecds:
+    input:
+        lambda w: expand(
+            resources("profile_pecd_{clusters}_{technology}.nc"),
+            **config["scenario"],
+            run=config["run"]["name"],
+            technology=config_provider(
+                "electricity", "pecd_renewable_profiles", "technologies"
+            )(w),
+        ),

scripts/_helpers.py

@@ -1129,3 +1129,48 @@ def extract_grid_data_tyndp(
     links.index = links.apply(make_index, axis=1, prefix=carrier)
 
     return links
+
+
+def safe_pyear(
+    year: int | str,
+    available_years: list = [2030, 2040, 2050],
+    source: str = "TYNDP",
+    verbose: bool = True,
+):
+    """
+    Checks and adjusts whether a given pyear is in the available years of a given data source. If not, it falls back to the previous available year.
+
+    Parameters
+    ----------
+    year : int
+        planning horizon year which will be checked and possibly adjusted to previous available year.
+    available_years : list, optional
+        List of available years. Defaults to [2030, 2040, 2050].
+    source : str, optional
+        Source of the data for which availability will be checked. For logging purpose only. Defaults to "TYNDP".
+    verbose : bool, optional
+        Whether to activate verbose logging. Defaults to True.
+
+    Returns
+    -------
+    year_new : int
+        safe pyear adjusted for available years
+    """
+
+    if not available_years:
+        raise ValueError(
+            "No `available_years` provided. Expected a non-empty list of years."
+        )
+    if not isinstance(year, int):
+        year = int(year)
+    if year not in available_years:
+        lower = [y for y in available_years if y < year]
+        year_new = max(lower) if lower else available_years[0]
+        if verbose:
+            logger.warning(
+                f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}."
+            )
+    else:
+        year_new = year
+
+    return year_new

scripts/add_existing_baseyear.py

@@ -102,6 +102,12 @@ def add_existing_renewables(
 
     irena = irena.unstack().reset_index()
 
+    if not set(tech_map.keys()).intersection(renewable_carriers):
+        # TODO: adjust once TYNDP existing capacities are added.
+        logger.info(
+            "No existing capacities to add for specified renewable carriers. Existing capacities of TYNDP renewable carriers will be added separately."
+        )
+
     for carrier, tech in tech_map.items():
         if carrier not in renewable_carriers:
             continue
@@ -145,7 +151,9 @@ def add_existing_renewables(
                     df_agg.at[name, "bus"] = bus
                     df_agg.at[name, "resource_class"] = bin_id
 
-    df_agg["resource_class"] = df_agg["resource_class"].fillna(0)
+    df_agg["resource_class"] = (
+        df_agg["resource_class"].fillna(0) if "resource_class" in df_agg.columns else 0
+    )
 
 
 def add_power_capacities_installed_before_baseyear(

scripts/build_renewable_profiles_pecd.py

@@ -22,12 +22,12 @@
 
 import logging
 
-import numpy as np
 import pandas as pd
 import xarray as xr
 
 from scripts._helpers import (
     configure_logging,
+    safe_pyear,
     set_scenario_config,
 )
 
@@ -55,16 +55,10 @@
             f"Extract PECD capacity factor time series for year {year} for technology {technology}..."
         )
         year_i = year
-        if int(year) not in [2030, 2040, 2050]:
-            year = np.clip(10 * (year // 10), 2030, 2050)
-            logger.warning(
-                f"TYNDP PECD data unavailable for planning horizon. Falling back to previous available year {year}."
-            )
-        if year == 2050:
-            logger.warning(
-                "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data."
-            )
-            year = 2040
+        # falling back to latest available pyear if not in list of available years
+        year = safe_pyear(
+            year, available_years=snakemake.params.available_years, source="PECD"
+        )
 
         profile = (
             pd.read_csv(