feat: process and prepare PECD capacity factor time series for offshore generators

Snakefile

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+from itertools import chain
 from pathlib import Path
 import yaml
 from os.path import normpath, exists, join

config/config.default.yaml

@@ -113,6 +113,19 @@ electricity:
   conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
   renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float, hydro]
 
+  pecd_renewable_profiles:
+    enable: false
+    technologies:
+      Wind_Offshore:
+      - offwind-ac-fb-r
+      - offwind-ac-fl-r
+      - offwind-dc-fb-r
+      - offwind-dc-fl-r
+      - offwind-dc-fb-oh
+      - offwind-dc-fl-oh
+      - offwind-h2-fb-oh
+      - offwind-h2-fl-oh
+
   estimate_renewable_capacities:
     enable: true
     from_gem: true

config/config.tyndp.yaml

@@ -40,6 +40,20 @@ co2_budget:
 electricity:
   base_network: tyndp-raw
   transmission_limit: v1.0
+  pecd_renewable_profiles:
+    enable: true
+    technologies:
+      Wind_Offshore:
+      - offwind-ac-fb-r
+      - offwind-ac-fl-r
+      - offwind-dc-fb-r
+      - offwind-dc-fl-r
+      - offwind-dc-fb-oh
+      - offwind-dc-fl-oh
+      - offwind-h2-fb-oh
+      - offwind-h2-fl-oh
+
+  renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro]
 
 links:
   p_max_pu: 1.0

config/test/config.tyndp.yaml

@@ -38,14 +38,26 @@ co2_budget:
 electricity:
   base_network: tyndp-raw
   transmission_limit: v1.0
+  pecd_renewable_profiles:
+    enable: true
+    technologies:
+      Wind_Offshore:
+      - offwind-ac-fb-r
+      - offwind-ac-fl-r
+      - offwind-dc-fb-r
+      - offwind-dc-fl-r
+      - offwind-dc-fb-oh
+      - offwind-dc-fl-oh
+      - offwind-h2-fb-oh
+      - offwind-h2-fl-oh
 
   extendable_carriers:
     Generator: [OCGT]
     StorageUnit: [battery]
     Store: [H2]
     Link: [H2 pipeline]
 
-  renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float]
+  renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
 
 atlite:
   default_cutout: europe-2013-03-sarah3-era5

doc/configtables/electricity.csv

@@ -30,6 +30,10 @@ everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignit
 conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound."
 ,,,
 renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model.
+pecd_renewable_profiles,,,
+-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_profile_name`` can be any of {Wind_Offshore, Wind_Onshore, LFSolarPV}.
+-- technologies,,,
+-- -- {pecd_profile_name},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. These TYNDP renewable carriers should also be listed in ``renewable_carriers``. Carriers not listed use default renewable profiles.
 estimate_renewable_capacities,,,
 -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`.
 -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker <https://globalenergymonitor.org/projects/global-solar-power-tracker/>`_ and `Global Energy Monitor's Global Wind Power Tracker <https://globalenergymonitor.org/projects/global-wind-power-tracker/>`_.

doc/release_notes.rst

@@ -13,6 +13,8 @@ Release Notes
 
 **Changes**
 
+* Add processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies.
+
 * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards.
 
 * Add the TYNDP electricity demand as an exogenously set demand (https://github.com/open-energy-transition/open-tyndp/pull/14). This requires the default PyPSA-Eur modelling to be explicitly disabled. The TYNDP electricity demand depends on the planning year, necessitating a different approach to the default PyPSA-Eur one. Wildcards are introduced and load is attached in `prepare_sector_network`.

rules/build_electricity.smk

@@ -393,6 +393,62 @@ rule build_renewable_profiles:
         "../scripts/build_renewable_profiles.py"
 
 
+rule clean_pecd_data:
+    params:
+        snapshots=config_provider("snapshots"),
+        drop_leap_day=config_provider("enable", "drop_leap_day"),
+    input:
+        offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx",
+        onshore_buses=resources("busmap_base_s_all.csv"),
+        dir_pecd="data/tyndp_2024_bundle/PECD",
+    output:
+        pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"),
+    log:
+        logs("clean_pecd_data_{technology}_{planning_horizons}.log"),
+    benchmark:
+        benchmarks("clean_pecd_data_{technology}_{planning_horizons}")
+    threads: 4
+    resources:
+        mem_mb=4000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/clean_pecd_data.py"
+
+
+def input_data_pecd(w):
+    return {
+        f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv")
+        for pyear in set(
+            config_provider("scenario", "planning_horizons")(w)
+        ).intersection([2030, 2040])
+        # Complete PECD data is only available for the years 2030, 2040
+        # TODO: adjust if udpated 2050 data available
+    }
+
+
+rule build_renewable_profiles_pecd:
+    params:
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+    input:
+        unpack(input_data_pecd),
+    output:
+        profile=resources("profile_pecd_{clusters}_{technology}.nc"),
+    log:
+        logs("build_renewable_profile_pecd_{clusters}_{technology}.log"),
+    benchmark:
+        benchmarks("build_renewable_profile_pecd_{clusters}_{technology}")
+    threads: 1
+    resources:
+        mem_mb=4000,
+    wildcard_constraints:
+        technology="(?!hydro).*",  # Any technology other than hydro
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_renewable_profiles_pecd.py"
+
+
 rule build_monthly_prices:
     input:
         co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls",
@@ -545,6 +601,7 @@ def input_class_regions(w):
         )
         for tech in set(config_provider("electricity", "renewable_carriers")(w))
         - {"hydro"}
+        - set(tyndp_renewable_carriers(w))
     }
 
 
@@ -729,14 +786,29 @@ rule cluster_network:
         "../scripts/cluster_network.py"
 
 
+def tyndp_renewable_carriers(w):
+    return (
+        list(
+            chain(
+                *config_provider(
+                    "electricity", "pecd_renewable_profiles", "technologies"
+                )(w).values()
+            )
+        )
+        if config_provider("electricity", "pecd_renewable_profiles", "enable")(w)
+        else []
+    )
+
+
 def input_profile_tech(w):
     return {
         f"profile_{tech}": resources(
             "profile_{clusters}_" + tech + ".nc"
             if tech != "hydro"
             else f"profile_{tech}.nc"
         )
-        for tech in config_provider("electricity", "renewable_carriers")(w)
+        for tech in set(config_provider("electricity", "renewable_carriers")(w))
+        - set(tyndp_renewable_carriers(w))
     }
 
 

rules/build_sector.smk

@@ -1228,6 +1228,27 @@ def input_profile_offwind(w):
         f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc")
         for tech in ["offwind-ac", "offwind-dc", "offwind-float"]
         if (tech in config_provider("electricity", "renewable_carriers")(w))
+        and (tech not in tyndp_renewable_carriers(w))
+    }
+
+
+def pecd_renewable_profiles(w):
+    return (
+        [
+            carrier
+            for carrier in config_provider(
+                "electricity", "pecd_renewable_profiles", "technologies"
+            )(w).keys()
+        ]
+        if config_provider("electricity", "pecd_renewable_profiles", "enable")(w)
+        else []
+    )
+
+
+def input_profile_pecd(w):
+    return {
+        f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc")
+        for tech in pecd_renewable_profiles(w)
     }
 
 
@@ -1379,6 +1400,7 @@ rule prepare_sector_network:
         scaling_factor=config_provider("load", "scaling_factor"),
     input:
         unpack(input_profile_offwind),
+        unpack(input_profile_pecd),
         unpack(input_heat_source_power),
         **rules.cluster_gas_network.output,
         **rules.build_gas_input_locations.output,

rules/retrieve.smk

@@ -185,6 +185,20 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle"
         script:
             "../scripts/retrieve_tyndp_bundle.py"
 
+    rule retrieve_tyndp_pecd_data:
+        params:
+            # TODO Integrate into Zenodo tyndp data bundle
+            tyndp_bundle="data/tyndp_2024_bundle",
+        output:
+            dir=directory("data/tyndp_2024_bundle/PECD"),
+        log:
+            "logs/retrieve_tyndp_pecd_data.log",
+        retries: 2
+        script:
+            "../scripts/retrieve_tyndp_pecd_data.py"
+
+    ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data
+
     rule retrieve_countries_centroids:
         output:
             "data/countries_centroids.geojson",

rules/solve_myopic.smk

@@ -7,7 +7,11 @@ rule add_existing_baseyear:
     params:
         baseyear=config_provider("scenario", "planning_horizons", 0),
         sector=config_provider("sector"),
+        pecd_renewable_profiles=config_provider(
+            "electricity", "pecd_renewable_profiles"
+        ),
         existing_capacities=config_provider("existing_capacities"),
+        carriers=config_provider("electricity", "renewable_carriers"),
         costs=config_provider("costs"),
         heat_pump_sources=config_provider("sector", "heat_pump_sources"),
         energy_totals_year=config_provider("energy", "energy_totals_year"),
@@ -59,11 +63,21 @@ rule add_existing_baseyear:
 def input_profile_tech_brownfield(w):
     return {
         f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc")
-        for tech in config_provider("electricity", "renewable_carriers")(w)
+        for tech in (
+            set(config_provider("electricity", "renewable_carriers")(w))
+            - set(tyndp_renewable_carriers(w))
+        )
         if tech != "hydro"
     }
 
 
+def input_profile_tech_brownfield_pecd(w):
+    return {
+        f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc")
+        for tech in pecd_renewable_profiles(w)
+    }
+
+
 rule add_brownfield:
     params:
         H2_retrofit=config_provider("sector", "H2_retrofit"),
@@ -72,6 +86,9 @@ rule add_brownfield:
         ),
         threshold_capacity=config_provider("existing_capacities", "threshold_capacity"),
         snapshots=config_provider("snapshots"),
+        pecd_renewable_profiles=config_provider(
+            "electricity", "pecd_renewable_profiles"
+        ),
         drop_leap_day=config_provider("enable", "drop_leap_day"),
         carriers=config_provider("electricity", "renewable_carriers"),
         heat_pump_sources=config_provider("sector", "heat_pump_sources"),
@@ -81,6 +98,7 @@ rule add_brownfield:
         ),
     input:
         unpack(input_profile_tech_brownfield),
+        unpack(input_profile_tech_brownfield_pecd),
         simplify_busmap=resources("busmap_base_s.csv"),
         cluster_busmap=resources("busmap_base_s_{clusters}.csv"),
         network=resources(

rules/solve_perfect.smk

@@ -5,7 +5,11 @@ rule add_existing_baseyear:
     params:
         baseyear=config_provider("scenario", "planning_horizons", 0),
         sector=config_provider("sector"),
+        pecd_renewable_profiles=config_provider(
+            "electricity", "pecd_renewable_profiles"
+        ),
         existing_capacities=config_provider("existing_capacities"),
+        carriers=config_provider("electricity", "renewable_carriers"),
         costs=config_provider("costs"),
         heat_pump_sources=config_provider("sector", "heat_pump_sources"),
         energy_totals_year=config_provider("energy", "energy_totals_year"),

scripts/add_brownfield.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>
+# SPDX-FileCopyrightText: Open Energy Transition gGmbH and contributors to PyPSA-Eur <https://github.com/pypsa/pypsa-eur>
 #
 # SPDX-License-Identifier: MIT
 """
@@ -19,7 +19,7 @@
     set_scenario_config,
     update_config_from_wildcards,
 )
-from scripts.add_electricity import flatten, sanitize_carriers
+from scripts.add_electricity import flatten, get_tyndp_res_carriers, sanitize_carriers
 from scripts.add_existing_baseyear import add_build_year_to_new_assets
 
 logger = logging.getLogger(__name__)
@@ -206,7 +206,9 @@ def disable_grid_expansion_if_limit_hit(n):
                 n.global_constraints.drop(name, inplace=True)
 
 
-def adjust_renewable_profiles(n, input_profiles, params, year):
+def adjust_renewable_profiles(
+    n, input_profiles, params, year, tyndp_renewable_carriers
+):
     """
     Adjusts renewable profiles according to the renewable technology specified,
     using the latest year below or equal to the selected year.
@@ -218,7 +220,12 @@ def adjust_renewable_profiles(n, input_profiles, params, year):
         pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill()
     )
 
-    for carrier in params["carriers"]:
+    # TODO: hotfix remove filter for tyndp_renewable_carriers after tyndp generators are added
+    if len(tyndp_renewable_carriers) > 0:
+        logger.info(
+            f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'."
+        )
+    for carrier in set(params["carriers"]) - set(tyndp_renewable_carriers):
         if carrier == "hydro":
             continue
 
@@ -349,7 +356,13 @@ def update_dynamic_ptes_capacity(
 
     n = pypsa.Network(snakemake.input.network)
 
-    adjust_renewable_profiles(n, snakemake.input, snakemake.params, year)
+    tyndp_renewable_carriers = get_tyndp_res_carriers(
+        snakemake.params.pecd_renewable_profiles
+    )
+
+    adjust_renewable_profiles(
+        n, snakemake.input, snakemake.params, year, tyndp_renewable_carriers
+    )
 
     add_build_year_to_new_assets(n, year)