feat: process and prepare PECD capacity factor time series for offshore generators

Snakefile

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+from itertools import chain
 from pathlib import Path
 import yaml
 from os.path import normpath, exists, join

config/config.default.yaml

@@ -116,7 +116,7 @@ electricity:
   pecd_renewable_profiles:
     enable: false
     technologies:
-      offwind:
+      Wind_Offshore:
       - offwind-ac-fb-r
       - offwind-ac-fl-r
       - offwind-dc-fb-r

config/config.tyndp.yaml

@@ -43,7 +43,7 @@ electricity:
   pecd_renewable_profiles:
     enable: true
     technologies:
-      offwind:
+      Wind_Offshore:
       - offwind-ac-fb-r
       - offwind-ac-fl-r
       - offwind-dc-fb-r

config/test/config.tyndp.yaml

@@ -41,7 +41,7 @@ electricity:
   pecd_renewable_profiles:
     enable: true
     technologies:
-      offwind:
+      Wind_Offshore:
       - offwind-ac-fb-r
       - offwind-ac-fl-r
       - offwind-dc-fb-r

doc/configtables/electricity.csv

@@ -31,9 +31,9 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite
 ,,,
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model.
 pecd_renewable_profiles,,,
--- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below.
+-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_profile_name`` can be any of {Wind_Offshore, Wind_Onshore, LFSolarPV}.
 -- technologies,,,
--- -- {pecd_profile},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. Carriers not listed use default renewable profiles.
+-- -- {pecd_profile_name},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. These TYNDP renewable carriers should also be listed in ``renewable_carriers``. Carriers not listed use default renewable profiles.
 estimate_renewable_capacities,,,
 -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`.
 -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker <https://globalenergymonitor.org/projects/global-solar-power-tracker/>`_ and `Global Energy Monitor's Global Wind Power Tracker <https://globalenergymonitor.org/projects/global-wind-power-tracker/>`_.

doc/release_notes.rst

@@ -13,7 +13,7 @@ Release Notes
 
 **Changes**
 
-* Add processing and preparation of TYNDP 2024 PECD renewable profiles instead of default renewable profiles using atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies.
+* Add processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies.
 
 * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards.
 

rules/build_electricity.smk

@@ -395,13 +395,12 @@ rule build_renewable_profiles:
 
 rule clean_pecd_data:
     params:
-        scenario=config_provider("tyndp_scenario"),
         snapshots=config_provider("snapshots"),
         drop_leap_day=config_provider("enable", "drop_leap_day"),
     input:
         offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx",
         onshore_buses=resources("busmap_base_s_all.csv"),
-        fn_pecd="data/tyndp_2024_bundle/PECD",
+        dir_pecd="data/tyndp_2024_bundle/PECD",
     output:
         pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"),
     log:
@@ -430,9 +429,6 @@ def input_data_pecd(w):
 
 rule build_renewable_profiles_pecd:
     params:
-        snapshots=config_provider("snapshots"),
-        drop_leap_day=config_provider("enable", "drop_leap_day"),
-        renewable=config_provider("renewable"),
         planning_horizons=config_provider("scenario", "planning_horizons"),
     input:
         unpack(input_data_pecd),
@@ -792,13 +788,13 @@ rule cluster_network:
 
 def tyndp_renewable_carriers(w):
     return (
-        [
-            subcarrier
-            for carrier in config_provider(
-                "electricity", "pecd_renewable_profiles", "technologies"
-            )(w).values()
-            for subcarrier in carrier
-        ]
+        list(
+            chain(
+                *config_provider(
+                    "electricity", "pecd_renewable_profiles", "technologies"
+                )(w).values()
+            )
+        )
         if config_provider("electricity", "pecd_renewable_profiles", "enable")(w)
         else []
     )

rules/retrieve.smk

@@ -187,8 +187,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle"
 
     rule retrieve_tyndp_pecd_data:
         params:
+            # TODO Integrate into Zenodo tyndp data bundle
             tyndp_bundle="data/tyndp_2024_bundle",
-        # TODO Integrate into Zenodo tyndp data bundle
         output:
             dir=directory("data/tyndp_2024_bundle/PECD"),
         log:

rules/solve_myopic.smk

@@ -7,7 +7,9 @@ rule add_existing_baseyear:
     params:
         baseyear=config_provider("scenario", "planning_horizons", 0),
         sector=config_provider("sector"),
-        electricity=config_provider("electricity"),
+        pecd_renewable_profiles=config_provider(
+            "electricity", "pecd_renewable_profiles"
+        ),
         existing_capacities=config_provider("existing_capacities"),
         carriers=config_provider("electricity", "renewable_carriers"),
         costs=config_provider("costs"),
@@ -69,7 +71,7 @@ def input_profile_tech_brownfield(w):
     }
 
 
-def input_profile_tech_brownfied_pecd(w):
+def input_profile_tech_brownfield_pecd(w):
     return {
         f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc")
         for tech in pecd_renewable_profiles(w)
@@ -84,7 +86,9 @@ rule add_brownfield:
         ),
         threshold_capacity=config_provider("existing_capacities", "threshold_capacity"),
         snapshots=config_provider("snapshots"),
-        electricity=config_provider("electricity"),
+        pecd_renewable_profiles=config_provider(
+            "electricity", "pecd_renewable_profiles"
+        ),
         drop_leap_day=config_provider("enable", "drop_leap_day"),
         carriers=config_provider("electricity", "renewable_carriers"),
         heat_pump_sources=config_provider("sector", "heat_pump_sources"),
@@ -94,7 +98,7 @@ rule add_brownfield:
         ),
     input:
         unpack(input_profile_tech_brownfield),
-        unpack(input_profile_tech_brownfied_pecd),
+        unpack(input_profile_tech_brownfield_pecd),
         simplify_busmap=resources("busmap_base_s.csv"),
         cluster_busmap=resources("busmap_base_s_{clusters}.csv"),
         network=resources(

rules/solve_perfect.smk

@@ -5,7 +5,9 @@ rule add_existing_baseyear:
     params:
         baseyear=config_provider("scenario", "planning_horizons", 0),
         sector=config_provider("sector"),
-        electricity=config_provider("electricity"),
+        pecd_renewable_profiles=config_provider(
+            "electricity", "pecd_renewable_profiles"
+        ),
         existing_capacities=config_provider("existing_capacities"),
         carriers=config_provider("electricity", "renewable_carriers"),
         costs=config_provider("costs"),

scripts/add_brownfield.py

@@ -19,7 +19,7 @@
     set_scenario_config,
     update_config_from_wildcards,
 )
-from scripts.add_electricity import flatten, sanitize_carriers
+from scripts.add_electricity import flatten, get_tyndp_res_carriers, sanitize_carriers
 from scripts.add_existing_baseyear import add_build_year_to_new_assets
 
 logger = logging.getLogger(__name__)
@@ -356,16 +356,8 @@ def update_dynamic_ptes_capacity(
 
     n = pypsa.Network(snakemake.input.network)
 
-    tyndp_renewable_carriers = (
-        [
-            subcarrier
-            for carrier in snakemake.params.electricity["pecd_renewable_profiles"][
-                "technologies"
-            ].values()
-            for subcarrier in carrier
-        ]
-        if snakemake.params.electricity["pecd_renewable_profiles"]["enable"]
-        else []
+    tyndp_renewable_carriers = get_tyndp_res_carriers(
+        snakemake.params.pecd_renewable_profiles
     )
 
     adjust_renewable_profiles(

scripts/add_electricity.py

@@ -50,6 +50,7 @@
 
 import logging
 from collections.abc import Iterable
+from itertools import chain
 from typing import Any
 
 import geopandas as gpd
@@ -189,6 +190,30 @@ def sanitize_carriers(n, config):
     n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors)
 
 
+def get_tyndp_res_carriers(pecd_renewable_profiles: dict):
+    """
+    Function to return all TYNDP renewable carriers specified in the configuration file for PECD profiles.
+
+    The function makes sure TYNDP renewable carriers are only returned if PECD profiles are enabled.
+
+    Parameters
+    ----------
+    pecd_renewable_profiles : dict
+        Dictionary that contains all TYNDP renewable carrier for of the PECD profiles.
+
+    Returns
+    -------
+    tyndp_renewable_carriers : list
+        List of TYNDP renewable carriers.
+    """
+    tyndp_renewable_carriers = (
+        list(chain(*pecd_renewable_profiles["technologies"].values()))
+        if pecd_renewable_profiles["enable"]
+        else []
+    )
+    return tyndp_renewable_carriers
+
+
 def sanitize_locations(n):
     if "location" in n.buses.columns:
         n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x))
@@ -1212,27 +1237,15 @@ def attach_stores(
         params.link_length_factor,
     )
 
-    tyndp_renewable_carriers = (
-        [
-            subcarrier
-            for carrier in params.electricity["pecd_renewable_profiles"][
-                "technologies"
-            ].values()
-            for subcarrier in carrier
-        ]
-        if params.electricity["pecd_renewable_profiles"]["enable"]
-        else []
+    tyndp_renewable_carriers = get_tyndp_res_carriers(
+        params.electricity["pecd_renewable_profiles"]
     )
     if len(tyndp_renewable_carriers) > 0:
         logger.info(
-            f"Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'. They will be attached later on with TYNDP data."
+            f"Skipping renewable carriers - they will be attached later with TYNDP data: {', '.join(tyndp_renewable_carriers)}"
         )
-    renewable_carriers = set(
-        [
-            carrier
-            for carrier in params.electricity["renewable_carriers"]
-            if carrier not in tyndp_renewable_carriers
-        ]
+    renewable_carriers = set(params.electricity["renewable_carriers"]).difference(
+        tyndp_renewable_carriers
     )
     extendable_carriers = params.electricity["extendable_carriers"]
     conventional_carriers = params.electricity["conventional_carriers"]

scripts/add_existing_baseyear.py

@@ -23,7 +23,11 @@
     set_scenario_config,
     update_config_from_wildcards,
 )
-from scripts.add_electricity import load_costs, sanitize_carriers
+from scripts.add_electricity import (
+    get_tyndp_res_carriers,
+    load_costs,
+    sanitize_carriers,
+)
 from scripts.build_energy_totals import cartesian
 from scripts.definitions.heat_system import HeatSystem
 from scripts.prepare_sector_network import cluster_heat_buses, define_spatial
@@ -109,46 +113,47 @@ def add_existing_renewables(
     irena = irena.unstack().reset_index()
 
     for carrier, tech in tech_map.items():
-        if carrier in renewable_carriers:
-            df = (
-                irena[irena.Technology.str.contains(tech)]
-                .drop(columns=["Technology"])
-                .set_index("Country")
-            )
-            df.columns = df.columns.astype(int)
-
-            # calculate yearly differences
-            df.insert(loc=0, value=0.0, column="1999")
-            df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0)
-
-            # distribute capacities among generators potential (p_nom_max)
-            gen_i = n.generators.query("carrier == @carrier").index
-            carrier_gens = n.generators.loc[gen_i]
-            res_capacities = []
-            for country, group in carrier_gens.groupby(
-                carrier_gens.bus.map(n.buses.country)
-            ):
-                fraction = group.p_nom_max / group.p_nom_max.sum()
-                res_capacities.append(cartesian(df.loc[country], fraction))
-            res_capacities = pd.concat(res_capacities, axis=1).T
-
-            for year in res_capacities.columns:
-                for gen in res_capacities.index:
-                    bus_bin = re.sub(f" {carrier}.*", "", gen)
-                    bus, bin_id = bus_bin.rsplit(" ", maxsplit=1)
-                    name = f"{bus_bin} {carrier}-{year}"
-                    capacity = res_capacities.loc[gen, year]
-                    if capacity > 0.0:
-                        cost_key = carrier.split("-", maxsplit=1)[0]
-                        df_agg.at[name, "Fueltype"] = carrier
-                        df_agg.at[name, "Capacity"] = capacity
-                        df_agg.at[name, "DateIn"] = year
-                        df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"]
-                        df_agg.at[name, "DateOut"] = (
-                            year + costs.at[cost_key, "lifetime"] - 1
-                        )
-                        df_agg.at[name, "bus"] = bus
-                        df_agg.at[name, "resource_class"] = bin_id
+        if carrier not in renewable_carriers:
+            continue
+        df = (
+            irena[irena.Technology.str.contains(tech)]
+            .drop(columns=["Technology"])
+            .set_index("Country")
+        )
+        df.columns = df.columns.astype(int)
+
+        # calculate yearly differences
+        df.insert(loc=0, value=0.0, column="1999")
+        df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0)
+
+        # distribute capacities among generators potential (p_nom_max)
+        gen_i = n.generators.query("carrier == @carrier").index
+        carrier_gens = n.generators.loc[gen_i]
+        res_capacities = []
+        for country, group in carrier_gens.groupby(
+            carrier_gens.bus.map(n.buses.country)
+        ):
+            fraction = group.p_nom_max / group.p_nom_max.sum()
+            res_capacities.append(cartesian(df.loc[country], fraction))
+        res_capacities = pd.concat(res_capacities, axis=1).T
+
+        for year in res_capacities.columns:
+            for gen in res_capacities.index:
+                bus_bin = re.sub(f" {carrier}.*", "", gen)
+                bus, bin_id = bus_bin.rsplit(" ", maxsplit=1)
+                name = f"{bus_bin} {carrier}-{year}"
+                capacity = res_capacities.loc[gen, year]
+                if capacity > 0.0:
+                    cost_key = carrier.split("-", maxsplit=1)[0]
+                    df_agg.at[name, "Fueltype"] = carrier
+                    df_agg.at[name, "Capacity"] = capacity
+                    df_agg.at[name, "DateIn"] = year
+                    df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"]
+                    df_agg.at[name, "DateOut"] = (
+                        year + costs.at[cost_key, "lifetime"] - 1
+                    )
+                    df_agg.at[name, "bus"] = bus
+                    df_agg.at[name, "resource_class"] = bin_id
 
     df_agg["resource_class"] = df_agg["resource_class"].fillna(0)
 
@@ -756,16 +761,8 @@ def add_heating_capacities_installed_before_baseyear(
     options = snakemake.params.sector
 
     renewable_carriers = snakemake.params.carriers
-    tyndp_renewable_carriers = (
-        [
-            subcarrier
-            for carrier in snakemake.params.electricity["pecd_renewable_profiles"][
-                "technologies"
-            ].values()
-            for subcarrier in carrier
-        ]
-        if snakemake.params.electricity["pecd_renewable_profiles"]["enable"]
-        else []
+    tyndp_renewable_carriers = get_tyndp_res_carriers(
+        snakemake.params.pecd_renewable_profiles
     )
 
     baseyear = snakemake.params.baseyear