feat: process and prepare PECD capacity factor time series for offshore generators

config/config.default.yaml

@@ -113,6 +113,13 @@ electricity:
   conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
   renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float, hydro]
 
+  tyndp_renewable_profiles:
+    enable: false
+    technologies:
+    - offwind-ac
+    - offwind-dc
+    - offwind-float
+
   estimate_renewable_capacities:
     enable: true
     from_gem: true

config/config.tyndp.yaml

@@ -40,6 +40,12 @@ co2_budget:
 electricity:
   base_network: tyndp-raw
   transmission_limit: v1.0
+  tyndp_renewable_profiles:
+    enable: true
+    technologies:
+    - offwind-ac
+    - offwind-dc
+    - offwind-float
 
 links:
   p_max_pu: 1.0

config/test/config.tyndp.yaml

@@ -37,6 +37,12 @@ co2_budget:
 electricity:
   base_network: tyndp-raw
   transmission_limit: v1.0
+  tyndp_renewable_profiles:
+    enable: true
+    technologies:
+    - offwind-ac
+    - offwind-dc
+    - offwind-float
 
   extendable_carriers:
     Generator: [OCGT]

doc/configtables/electricity.csv

@@ -30,6 +30,9 @@ everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignit
 conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound."
 ,,,
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model.
+tyndp_renewable_profiles,,,
+-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below.
+-- technologies,--,list,Select the renewable technologies for which PECD renewable profiles are used. Carriers not listed use default renewable profiles.
 estimate_renewable_capacities,,,
 -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`.
 -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker <https://globalenergymonitor.org/projects/global-solar-power-tracker/>`_ and `Global Energy Monitor's Global Wind Power Tracker <https://globalenergymonitor.org/projects/global-wind-power-tracker/>`_.

doc/release_notes.rst

@@ -13,6 +13,8 @@ Release Notes
 
 **Changes**
 
+* Add processing and preparation of TYNDP 2024 PECD renewable profiles instead of default renewable profiles using atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies.
+
 * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards.
 
 * Add the TYNDP electricity demand as an exogenously set demand (https://github.com/open-energy-transition/open-tyndp/pull/14). This requires the default PyPSA-Eur modelling to be explicitly disabled. The TYNDP electricity demand depends on the planning year, necessitating a different approach to the default PyPSA-Eur one. Wildcards are introduced and load is attached in `prepare_sector_network`.

rules/build_electricity.smk

@@ -393,6 +393,65 @@ rule build_renewable_profiles:
         "../scripts/build_renewable_profiles.py"
 
 
+def input_data_pecd(w):
+    return {
+        f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv")
+        for pyear in set(
+            config_provider("scenario", "planning_horizons")(w)
+        ).intersection([2030, 2040])
+        # Complete PECD data is only available for the years 2030, 2040
+        # TODO: adjust if udpated 2050 data available
+    }
+
+
+rule build_renewable_profiles_pecd:
+    params:
+        snapshots=config_provider("snapshots"),
+        drop_leap_day=config_provider("enable", "drop_leap_day"),
+        renewable=config_provider("renewable"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+    input:
+        unpack(input_data_pecd),
+    output:
+        profile=resources("profile_pecd_{clusters}_{technology}.nc"),
+    log:
+        logs("build_renewable_profile_pecd_{clusters}_{technology}.log"),
+    benchmark:
+        benchmarks("build_renewable_profile_pecd_{clusters}_{technology}")
+    threads: 1
+    resources:
+        mem_mb=4000,
+    wildcard_constraints:
+        technology="(?!hydro).*",  # Any technology other than hydro
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_renewable_profiles_pecd.py"
+
+
+rule clean_pecd_data:
+    params:
+        scenario=config_provider("tyndp_scenario"),
+        snapshots=config_provider("snapshots"),
+    input:
+        offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx",
+        onshore_buses=resources("busmap_base_s_all.csv"),
+        fn_pecd="data/tyndp_2024_bundle/PECD",
+    output:
+        pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"),
+    log:
+        logs("clean_pecd_data_{technology}_{planning_horizons}.log"),
+    benchmark:
+        benchmarks("clean_pecd_data_{technology}_{planning_horizons}")
+    threads: 4
+    resources:
+        mem_mb=4000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/clean_pecd_data.py"
+
+
 rule build_monthly_prices:
     input:
         co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls",
@@ -545,6 +604,7 @@ def input_class_regions(w):
         )
         for tech in set(config_provider("electricity", "renewable_carriers")(w))
         - {"hydro"}
+        - set(tyndp_renewable_profiles(w))
     }
 
 
@@ -729,14 +789,23 @@ rule cluster_network:
         "../scripts/cluster_network.py"
 
 
+def tyndp_renewable_profiles(w):
+    return (
+        config_provider("electricity", "tyndp_renewable_profiles", "technologies")(w)
+        if config_provider("electricity", "tyndp_renewable_profiles", "enable")(w)
+        else []
+    )
+
+
 def input_profile_tech(w):
     return {
         f"profile_{tech}": resources(
             "profile_{clusters}_" + tech + ".nc"
             if tech != "hydro"
             else f"profile_{tech}.nc"
         )
-        for tech in config_provider("electricity", "renewable_carriers")(w)
+        for tech in set(config_provider("electricity", "renewable_carriers")(w))
+        - set(tyndp_renewable_profiles(w))
     }
 
 

rules/build_sector.smk

@@ -1228,6 +1228,14 @@ def input_profile_offwind(w):
         f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc")
         for tech in ["offwind-ac", "offwind-dc", "offwind-float"]
         if (tech in config_provider("electricity", "renewable_carriers")(w))
+        and (tech not in tyndp_renewable_profiles(w))
+    }
+
+
+def input_profile_pecd(w):
+    return {
+        f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc")
+        for tech in tyndp_renewable_profiles(w)
     }
 
 
@@ -1379,6 +1387,7 @@ rule prepare_sector_network:
         scaling_factor=config_provider("load", "scaling_factor"),
     input:
         unpack(input_profile_offwind),
+        unpack(input_profile_pecd),
         unpack(input_heat_source_power),
         **rules.cluster_gas_network.output,
         **rules.build_gas_input_locations.output,

rules/retrieve.smk

@@ -185,6 +185,20 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle"
         script:
             "../scripts/retrieve_tyndp_bundle.py"
 
+    rule retrieve_tyndp_pecd_data:
+        params:
+            tyndp_bundle="data/tyndp_2024_bundle",
+        # TODO Integrate into Zenodo tyndp data bundle
+        output:
+            dir=directory("data/tyndp_2024_bundle/PECD"),
+        log:
+            "logs/retrieve_tyndp_pecd_data.log",
+        retries: 2
+        script:
+            "../scripts/retrieve_tyndp_pecd_data.py"
+
+    ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data
+
     rule retrieve_countries_centroids:
         output:
             "data/countries_centroids.geojson",

rules/solve_myopic.smk

@@ -7,6 +7,7 @@ rule add_existing_baseyear:
     params:
         baseyear=config_provider("scenario", "planning_horizons", 0),
         sector=config_provider("sector"),
+        electricity=config_provider("electricity"),
         existing_capacities=config_provider("existing_capacities"),
         costs=config_provider("costs"),
         heat_pump_sources=config_provider("sector", "heat_pump_sources"),
@@ -59,11 +60,21 @@ rule add_existing_baseyear:
 def input_profile_tech_brownfield(w):
     return {
         f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc")
-        for tech in config_provider("electricity", "renewable_carriers")(w)
+        for tech in (
+            set(config_provider("electricity", "renewable_carriers")(w))
+            - set(tyndp_renewable_profiles(w))
+        )
         if tech != "hydro"
     }
 
 
+def input_profile_tech_brownfied_pecd(w):
+    return {
+        f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc")
+        for tech in tyndp_renewable_profiles(w)
+    }
+
+
 rule add_brownfield:
     params:
         H2_retrofit=config_provider("sector", "H2_retrofit"),
@@ -72,6 +83,7 @@ rule add_brownfield:
         ),
         threshold_capacity=config_provider("existing_capacities", "threshold_capacity"),
         snapshots=config_provider("snapshots"),
+        electricity=config_provider("electricity"),
         drop_leap_day=config_provider("enable", "drop_leap_day"),
         carriers=config_provider("electricity", "renewable_carriers"),
         heat_pump_sources=config_provider("sector", "heat_pump_sources"),
@@ -81,6 +93,7 @@ rule add_brownfield:
         ),
     input:
         unpack(input_profile_tech_brownfield),
+        unpack(input_profile_tech_brownfied_pecd),
         simplify_busmap=resources("busmap_base_s.csv"),
         cluster_busmap=resources("busmap_base_s_{clusters}.csv"),
         network=resources(

rules/solve_perfect.smk

@@ -5,6 +5,7 @@ rule add_existing_baseyear:
     params:
         baseyear=config_provider("scenario", "planning_horizons", 0),
         sector=config_provider("sector"),
+        electricity=config_provider("electricity"),
         existing_capacities=config_provider("existing_capacities"),
         costs=config_provider("costs"),
         heat_pump_sources=config_provider("sector", "heat_pump_sources"),

scripts/add_brownfield.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>
+# SPDX-FileCopyrightText: Open Energy Transition gGmbH and contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>
 #
 # SPDX-License-Identifier: MIT
 """
@@ -218,7 +218,10 @@ def adjust_renewable_profiles(n, input_profiles, params, year):
         pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill()
     )
 
-    for carrier in params["carriers"]:
+    # TODO: hotfix remove filter for tyndp_renewable_profiles after tyndp generators are added
+    for carrier in set(params["carriers"]) - set(
+        params["electricity"]["tyndp_renewable_profiles"]["technologies"]
+    ):
         if carrier == "hydro":
             continue
 

scripts/add_electricity.py

@@ -1212,7 +1212,22 @@ def attach_stores(
         params.link_length_factor,
     )
 
-    renewable_carriers = set(params.electricity["renewable_carriers"])
+    tyndp_renewable_profiles = (
+        params.electricity["tyndp_renewable_profiles"]["technologies"]
+        if params.electricity["tyndp_renewable_profiles"]["enable"]
+        else []
+    )
+    if len(tyndp_renewable_profiles) > 0:
+        logger.info(
+            f"Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'. They will be attached later on with TYNDP data."
+        )
+    renewable_carriers = set(
+        [
+            carrier
+            for carrier in params.electricity["renewable_carriers"]
+            if carrier not in tyndp_renewable_profiles
+        ]
+    )
     extendable_carriers = params.electricity["extendable_carriers"]
     conventional_carriers = params.electricity["conventional_carriers"]
     conventional_inputs = {
@@ -1276,7 +1291,11 @@ def attach_stores(
                 "in rule `add_existing_baseyear` with foresight mode 'myopic'."
             )
         else:
-            tech_map = estimate_renewable_caps["technology_mapping"]
+            tech_map = {
+                key: value
+                for key, value in estimate_renewable_caps["technology_mapping"].items()
+                if value not in tyndp_renewable_profiles
+            }
             expansion_limit = estimate_renewable_caps["expansion_limit"]
             year = estimate_renewable_caps["year"]
 

scripts/add_existing_baseyear.py

@@ -68,6 +68,7 @@ def add_existing_renewables(
     costs: pd.DataFrame,
     df_agg: pd.DataFrame,
     countries: list[str],
+    tyndp_renewable_profiles: list[str],
 ) -> None:
     """
     Add existing renewable capacities to conventional power plant data.
@@ -82,13 +83,22 @@ def add_existing_renewables(
         Network containing topology and generator data
     countries : list
         List of country codes to consider
+    tyndp_renewable_profiles: list
+        List of renewable profile technologies taken from tyndp PECD
 
     Returns
     -------
     None
         Modifies df_agg in-place
     """
     tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"}
+    # TODO: remove when TYNDP renewable generators are added
+    if len(tyndp_renewable_profiles) > 0:
+        logger.info(
+            f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'."
+        )
+        for k in tyndp_renewable_profiles:
+            tech_map.pop(k, None)
 
     irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2()
     irena = irena.query("Country in @countries")
@@ -149,6 +159,7 @@ def add_power_capacities_installed_before_baseyear(
     countries: list[str],
     capacity_threshold: float,
     lifetime_values: dict[str, float],
+    tyndp_renewable_profiles: list[str],
 ) -> None:
     """
     Add power generation capacities installed before base year.
@@ -171,6 +182,8 @@ def add_power_capacities_installed_before_baseyear(
         Minimum capacity threshold
     lifetime_values : dict
         Default values for missing data
+    tyndp_renewable_profiles: list
+        List of renewable profile technologies taken from tyndp PECD
     """
     logger.debug(f"Adding power capacities installed before {baseyear}")
 
@@ -223,6 +236,7 @@ def add_power_capacities_installed_before_baseyear(
         costs=costs,
         n=n,
         countries=countries,
+        tyndp_renewable_profiles=tyndp_renewable_profiles,
     )
     # drop assets which are already phased out / decommissioned
     phased_out = df_agg[df_agg["DateOut"] < baseyear].index
@@ -727,6 +741,12 @@ def add_heating_capacities_installed_before_baseyear(
 
     options = snakemake.params.sector
 
+    tyndp_renewable_profiles = (
+        snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"]
+        if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"]
+        else []
+    )
+
     baseyear = snakemake.params.baseyear
 
     n = pypsa.Network(snakemake.input.network)
@@ -753,6 +773,7 @@ def add_heating_capacities_installed_before_baseyear(
         countries=snakemake.config["countries"],
         capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"],
         lifetime_values=snakemake.params.costs["fill_values"],
+        tyndp_renewable_profiles=tyndp_renewable_profiles,
     )
 
     if options["heating"]: