feat: Integrate missing PECD data

config/config.default.yaml

@@ -145,6 +145,10 @@ electricity:
   pecd_renewable_profiles:
     enable: false
     fill_gaps_method: zero
+    available_years:
+    - 2030
+    - 2040
+    - 2050
     technologies:
       Wind_Offshore:
       - offwind-ac-fb-r
@@ -393,6 +397,10 @@ load:
     enable: true
     interpolate_limit: 3
     time_shift_for_large_gaps: 1w
+  available_years_tyndp:
+  - 2030
+  - 2040
+  - 2050
   manual_adjustments: true
   scaling_factor: 1.0
   fixed_year: false

config/config.tyndp.yaml

@@ -37,12 +37,18 @@ co2_budget:
 electricity:
   base_network: tyndp
 
+  # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
   renewable_carriers: [onwind, solar, solar-hsat, hydro]
   tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
 
   pecd_renewable_profiles:
     enable: true
     fill_gaps_method: zero
+    # Complete PECD data is only available for the years 2030, 2040
+    # TODO: adjust once udpated 2050 PECD data ist available
+    available_years:
+    - 2030
+    - 2040
     technologies:
       Wind_Offshore:
       - offwind-ac-fb-r
@@ -83,6 +89,10 @@ load:
   source: tyndp # opsd, tyndp
   fill_gaps:
     enable: false
+  available_years_tyndp:
+  - 2030
+  - 2040
+  - 2050
   manual_adjustments: false
   supplement_synthetic: false
 

config/test/config.tyndp.yaml

@@ -48,12 +48,18 @@ electricity:
     Store: [H2]
     Link: [H2 pipeline]
 
+  # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
   renewable_carriers: [onwind, solar, solar-hsat]
   tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
 
   pecd_renewable_profiles:
     enable: true
     fill_gaps_method: zero
+    # Complete PECD data is only available for the years 2030, 2040
+    # TODO: adjust once udpated 2050 PECD data ist available
+    available_years:
+    - 2030
+    - 2040
     technologies:
       Wind_Offshore:
       - offwind-ac-fb-r
@@ -109,6 +115,10 @@ load:
   source: tyndp # opsd, tyndp
   fill_gaps:
     enable: false
+  available_years_tyndp:
+  - 2030
+  - 2040
+  - 2050
   manual_adjustments: false
   supplement_synthetic: false
 

doc/configtables/electricity.csv

@@ -32,8 +32,9 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model.
 tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv-rooftop, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh}",List of TYNDP renewable generators to include in the model. Technologies covered by specified TYNDP renewable carriers need to be removed from `estimate_renewable_carriers` technology list.
 pecd_renewable_profiles,,,
--- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPVUtility, LFSolarPVRooftop, Wind_Offshore, Wind_Onshore}.
+-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {LFSolarPVUtility, LFSolarPVRooftop, Wind_Offshore, Wind_Onshore, CSP_noStorage, CSP_withStorage_7h_dispatched, CSP_withStorage_7h_preDispatch}.
 -- fill_gaps_method,,str,The chosen method for filling gaps in the PECD data to the modelled nodes. Can be either `zero` to fill with zero values or any other aggregation method such as `mean`, `median`, `max` or similar.
+-- available_years,--,list,"List of years for which PECD data is available."
 -- technologies,,,
 -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``.
 -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles.

doc/configtables/load.csv

@@ -4,6 +4,7 @@ fill_gaps,--,--,"Gaps filling strategy used."
 -- enable,bool,"{true, false}","Whether to fill gaps using interpolation for small gaps and time shift for large gaps."
 -- interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly."
 -- time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings."
+available_years_tyndp,--,list,"List of years for which TYNDP demand data is available."
 manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`."
 scaling_factor,--,float,"Global correction factor for the load time series."
 fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year"

doc/release_notes.rst

@@ -11,7 +11,7 @@ Release Notes
 Upcoming Open-TYNDP Release
 ================
 
-
+* Add complete compatibility for processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP), replacing default ERA5- and SARAH3-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/71). This implementation serves to facilitate a sub-workflow for creation of the renewable profiles but does not yet attach them to any technologies yet.
 
 
 Upcoming PyPSA-Eur Release

rules/build_electricity.smk

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: MIT
 
+from scripts._helpers import safe_pyear
+
 
 def input_elec_demand(w):
     return {
@@ -400,6 +402,9 @@ rule clean_pecd_data:
         fill_gaps_method=config_provider(
             "electricity", "pecd_renewable_profiles", "fill_gaps_method"
         ),
+        available_years=config_provider(
+            "electricity", "pecd_renewable_profiles", "available_years"
+        ),
     input:
         offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx",
         onshore_buses=resources("busmap_base_s_all.csv"),
@@ -423,16 +428,27 @@ def input_data_pecd(w):
     return {
         f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv")
         for pyear in set(
-            config_provider("scenario", "planning_horizons")(w)
-        ).intersection([2030, 2040])
-        # Complete PECD data is only available for the years 2030, 2040
-        # TODO: adjust if udpated 2050 data available
+            [
+                safe_pyear(
+                    year,
+                    config_provider(
+                        "electricity", "pecd_renewable_profiles", "available_years"
+                    )(w),
+                    "PECD",
+                    verbose=False,
+                )
+                for year in config_provider("scenario", "planning_horizons")(w)
+            ]
+        )
     }
 
 
 rule build_renewable_profiles_pecd:
     params:
         planning_horizons=config_provider("scenario", "planning_horizons"),
+        available_years=config_provider(
+            "electricity", "pecd_renewable_profiles", "available_years"
+        ),
     input:
         unpack(input_data_pecd),
     output:
@@ -1034,6 +1050,7 @@ if config["load"]["source"] == "tyndp":
             planning_horizons=config_provider("scenario", "planning_horizons"),
             snapshots=config_provider("snapshots"),
             scenario=config_provider("tyndp_scenario"),
+            available_years=config_provider("load", "available_years_tyndp"),
         input:
             electricity_demand=directory("data/tyndp_2024_bundle/Demand Profiles"),
         output:

rules/collect.smk

@@ -90,3 +90,27 @@ rule plot_power_networks_clustered:
             **config["scenario"],
             run=config["run"]["name"],
         ),
+
+
+rule clean_pecd_datas:
+    input:
+        lambda w: expand(
+            resources("pecd_data_{technology}_{planning_horizons}.csv"),
+            **config["scenario"],
+            run=config["run"]["name"],
+            technology=config_provider(
+                "electricity", "pecd_renewable_profiles", "technologies"
+            )(w),
+        ),
+
+
+rule build_renewable_profiles_pecds:
+    input:
+        lambda w: expand(
+            resources("profile_pecd_{clusters}_{technology}.nc"),
+            **config["scenario"],
+            run=config["run"]["name"],
+            technology=config_provider(
+                "electricity", "pecd_renewable_profiles", "technologies"
+            )(w),
+        ),

scripts/_helpers.py

@@ -1129,3 +1129,44 @@ def extract_grid_data_tyndp(
     links.index = links.apply(make_index, axis=1, prefix=carrier)
 
     return links
+
+
+def safe_pyear(
+    year: int,
+    available_years: list = [2030, 2040, 2050],
+    source: str = "TYNDP",
+    verbose: bool = True,
+):
+    """
+    Checks and adjusts whether a given pyear is in the available years and falls back to the previous available year.
+
+    Parameters
+    ----------
+    year : int
+        planning horizon year which will be checked and possibly adjusted to previous available year
+    available_years : list, optional
+        list of available years. Defaults to [2030, 2040, 2050]
+    source : str, optional
+        source of the data for which availability will be checked. Defaults to "TYNDP"
+    verbose : bool, optional
+        Whether to activate verbose logging. Defaults to True
+
+    Returns
+    -------
+    year_new : str
+        safe pyear as a string
+    """
+
+    if not available_years:
+        raise ValueError("`available_years` cannot be empty.")
+    if year not in available_years:
+        lower = [y for y in available_years if y < year]
+        year_new = max(lower) if lower else available_years[0]
+        if verbose:
+            logger.warning(
+                f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}."
+            )
+    else:
+        year_new = year
+
+    return str(year_new)

scripts/add_existing_baseyear.py

@@ -103,7 +103,10 @@ def add_existing_renewables(
     irena = irena.unstack().reset_index()
 
     if not set(tech_map.keys()).intersection(renewable_carriers):
-        logger.info("No existing capacities to add for specified renewable carriers.")
+        # TODO: adjust once TYNDP existing capacities are added.
+        logger.info(
+            "No existing capacities to add for specified renewable carriers. Existing capacities of TYNDP renewable carriers will be added separately."
+        )
 
     for carrier, tech in tech_map.items():
         if carrier not in renewable_carriers:

scripts/build_renewable_profiles_pecd.py

@@ -22,12 +22,12 @@
 
 import logging
 
-import numpy as np
 import pandas as pd
 import xarray as xr
 
 from scripts._helpers import (
     configure_logging,
+    safe_pyear,
     set_scenario_config,
 )
 
@@ -55,11 +55,11 @@
             f"Extract PECD capacity factor time series for year {year} for technology {technology}..."
         )
         year_i = year
-        if int(year) not in [2030, 2040, 2050]:
-            year = np.clip(10 * (year // 10), 2030, 2050)
-            logger.warning(
-                f"TYNDP PECD data unavailable for planning horizon. Falling back to previous available year {year}."
-            )
+        # falling back to latest available pyear if not in list of available years
+        year = safe_pyear(
+            int(year), available_years=snakemake.params.available_years, source="PECD"
+        )
+        # TODO: remove once PECD data is updated
         if year == 2050:
             logger.warning(
                 "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data."

scripts/clean_pecd_data.py

@@ -3,13 +3,15 @@
 # SPDX-License-Identifier: MIT
 """
 Loads and cleans the available PECD capacity factor generation time series based on PECD weather data.
-The script is executed for a given technology, and planning horizon. Technologies can be one of:
+The script is executed for a given technology and planning horizon. Technologies can be one of:
 
-   * CSP
    * LFSolarPVUtility,
    * LFSolarPVRooftop,
    * Wind_Offshore,
-   * Wind_Onshore.
+   * Wind_Onshore,
+   * CSP_noStorage,
+   * CSP_withStorage_7h_dispatched,
+   * CSP_withStorage_7h_preDispatch (note: includes cf > 1 for when thermal storage can be used).
 
 Outputs
 -------
@@ -28,6 +30,7 @@
 from scripts._helpers import (
     configure_logging,
     get_snapshots,
+    safe_pyear,
     set_scenario_config,
 )
 
@@ -42,17 +45,28 @@ def read_pecd_file(
     technology: str,
     sns: pd.DatetimeIndex,
 ):
-    fn = Path(dir_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv")
+    fn = Path(
+        dir_pecd,
+        pyear,
+        f"PECD_{technology}_{pyear}_{node.replace('GB', 'UK')}_edition 2023.2.csv",
+    )
 
     # PECD only differentiates between utility and rooftop PV for some nodes
     if not os.path.isfile(fn) and "LFSolarPV" in technology:
         fn = Path(str(fn).replace(technology, "LFSolarPV"))
     if not os.path.isfile(fn):
+        logger.warning(f"Missing data for {technology} in {node} in {pyear}.")
         return None
 
+    # Malta CSP data file has an extra header row that must be skipped
+    if node == "MT00" and technology == "CSP_noStorage" and pyear == "2040":
+        skiprows = 11
+    else:
+        skiprows = 10
+
     pecd_bus = pd.read_csv(
         fn,
-        skiprows=10,  # first ten rows contain only file metadata
+        skiprows=skiprows,  # first rows contain only file metadata
         usecols=lambda name: name == "Date"
         or name == "Hour"
         or name == str(cyear)
@@ -95,8 +109,12 @@ def read_pecd_file(
         )
         cyear = 2009
 
-    # Planning year
-    pyear = str(snakemake.wildcards.planning_horizons)
+    # Planning year (falls back to latest available pyear if not in list of available years)
+    pyear = safe_pyear(
+        int(snakemake.wildcards.planning_horizons),
+        available_years=snakemake.params.available_years,
+        source="PECD",
+    )
 
     # Technology as in PECD terminology
     pecd_tech = snakemake.wildcards.technology
@@ -105,9 +123,11 @@ def read_pecd_file(
     onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0)
 
     nodes = (
-        offshore_buses.index
+        offshore_buses.index.str.replace(
+            "UK", "GB", regex=True
+        )  # replace UK with GB for naming convention
         if pecd_tech == "Wind_Offshore"
-        else onshore_buses.index.str.replace("GB", "UK", regex=True)
+        else onshore_buses.index
     )
     dir_pecd = snakemake.input.dir_pecd
 
@@ -133,7 +153,7 @@ def read_pecd_file(
 
     if all(data is None for data in pecd):
         raise ValueError(
-            f"No PECD data found for {pecd_tech}. Please specify a technology covered within the TYNDP PECD data."
+            f"No PECD data found for {pecd_tech} in {pyear}. Please specify a technology covered within the TYNDP PECD data."
         )
     pecd_df = pd.concat(pecd, axis=1)
     fill_na = (
@@ -142,11 +162,9 @@ def read_pecd_file(
         else pecd_df.agg(snakemake.params.fill_gaps_method, axis=1)
     )
     pecd_df = (
-        pecd_df.reindex(nodes, axis=1)  # include missing node data with empty columns
-        .rename(
-            columns=lambda x: x.replace("UK", "GB")
-        )  # replace UK with GB for naming convention
-        .where(
+        pecd_df.reindex(
+            nodes, axis=1
+        ).where(  # include missing node data with empty columns
             lambda df: df.notna(), fill_na, axis=0
         )  # fill missing node data with configured aggregation method
     )