From 1cf34a9b730976949ac3cb604bdb4414a491a664 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 23 May 2025 17:39:38 +0200 Subject: [PATCH 01/49] feat: skip tyndp_renewable_profile technologies in workflow and build_renewable_profiles --- config/config.default.yaml | 7 +++++++ config/config.tyndp.yaml | 6 ++++++ config/test/config.tyndp.yaml | 6 ++++++ rules/build_electricity.smk | 12 +++++++++++- rules/build_sector.smk | 1 + rules/solve_myopic.smk | 5 ++++- scripts/add_electricity.py | 23 +++++++++++++++++++++-- 7 files changed, 56 insertions(+), 4 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index da6c69d146..234b0ecabc 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -111,6 +111,13 @@ electricity: conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float, hydro] + tyndp_renewable_profiles: + enable: false + technologies: + - offwind-ac + - offwind-dc + - offwind-float + estimate_renewable_capacities: enable: true from_gem: true diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index e151f88b43..8a0836ec6c 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -38,6 +38,12 @@ co2_budget: electricity: base_network: tyndp-raw transmission_limit: v1.0 + tyndp_renewable_profiles: + enable: true + technologies: + - offwind-ac + - offwind-dc + - offwind-float links: p_max_pu: 1.0 diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 77f3498685..4cbd874a44 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -35,6 +35,12 @@ co2_budget: electricity: base_network: tyndp-raw transmission_limit: v1.0 + tyndp_renewable_profiles: + enable: true + technologies: + - offwind-ac + - offwind-dc + - offwind-float extendable_carriers: Generator: [OCGT] diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 673b07a770..5966f7d2ee 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -540,6 +540,7 @@ def input_class_regions(w): ) for tech in set(config_provider("electricity", "renewable_carriers")(w)) - {"hydro"} + - set(tyndp_renewable_profiles(w)) } @@ -705,6 +706,14 @@ rule cluster_network: "../scripts/cluster_network.py" +def tyndp_renewable_profiles(w): + return ( + config_provider("electricity", "tyndp_renewable_profiles", "technologies")(w) + if config_provider("electricity", "tyndp_renewable_profiles", "enable")(w) + else [] + ) + + def input_profile_tech(w): return { f"profile_{tech}": resources( @@ -712,7 +721,8 @@ def input_profile_tech(w): if tech != "hydro" else f"profile_{tech}.nc" ) - for tech in config_provider("electricity", "renewable_carriers")(w) + for tech in set(config_provider("electricity", "renewable_carriers")(w)) + - set(tyndp_renewable_profiles(w)) } diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 3dbc4afd0f..b56071a4df 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1226,6 +1226,7 @@ def input_profile_offwind(w): f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") for tech in ["offwind-ac", "offwind-dc", "offwind-float"] if (tech in config_provider("electricity", "renewable_carriers")(w)) + and (tech not in tyndp_renewable_profiles(w)) } diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 87379b08b6..e2068bc817 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -59,7 +59,10 @@ rule add_existing_baseyear: def input_profile_tech_brownfield(w): return { f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") - for tech in config_provider("electricity", "renewable_carriers")(w) + for tech in ( + set(config_provider("electricity", "renewable_carriers")(w)) + - set(tyndp_renewable_profiles(w)) + ) if tech != "hydro" } diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 0a64e364a8..d7c9a314cb 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1203,7 +1203,22 @@ def attach_stores( params.link_length_factor, ) - renewable_carriers = set(params.electricity["renewable_carriers"]) + tyndp_renewable_profiles = ( + params.electricity["tyndp_renewable_profiles"]["technologies"] + if params.electricity["tyndp_renewable_profiles"]["enable"] + else [] + ) + if len(tyndp_renewable_profiles) > 0: + logger.info( + f"Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'. They will be attached later on with TYNDP data." + ) + renewable_carriers = set( + [ + carrier + for carrier in params.electricity["renewable_carriers"] + if carrier not in tyndp_renewable_profiles + ] + ) extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { @@ -1267,7 +1282,11 @@ def attach_stores( "in rule `add_existing_baseyear` with foresight mode 'myopic'." ) else: - tech_map = estimate_renewable_caps["technology_mapping"] + tech_map = { + key: value + for key, value in estimate_renewable_caps["technology_mapping"].items() + if value not in tyndp_renewable_profiles + } expansion_limit = estimate_renewable_caps["expansion_limit"] year = estimate_renewable_caps["year"] From 3be60a1be9fa1b456a2a27f6ca7dfbbc90b8f0f2 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 14:34:02 +0200 Subject: [PATCH 02/49] feat: clean pecd data for years 2030, 2040, 2050 and build_renewable_profiles_pecd --- rules/build_electricity.smk | 70 ++++++++++++ scripts/build_renewable_profiles_pecd.py | 93 ++++++++++++++++ scripts/clean_pecd_data.py | 135 +++++++++++++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 scripts/build_renewable_profiles_pecd.py create mode 100644 scripts/clean_pecd_data.py diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 5966f7d2ee..614cb0e112 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -381,6 +381,76 @@ rule build_renewable_profiles: "../scripts/build_renewable_profiles.py" +def input_data_pecd(w): + return { + f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + pyear + ".csv") + for pyear in [ + "2030", + "2040", + "2050", + ] # complete PECD data is available for the years 2030, 2040 + } + + +rule build_renewable_profiles_pecd: + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + renewable=config_provider("renewable"), + planning_horizons=config_provider("scenario", "planning_horizons"), + input: + unpack(input_data_pecd), + availability_matrix=resources("availability_matrix_{clusters}_{technology}.nc"), + offshore_shapes=resources("offshore_shapes.geojson"), + distance_regions=resources("regions_onshore_base_s_{clusters}.geojson"), + resource_regions=lambda w: ( + resources("regions_onshore_base_s_{clusters}.geojson") + if w.technology in ("onwind", "solar", "solar-hsat") + else resources("regions_offshore_base_s_{clusters}.geojson") + ), + cutout=lambda w: input_cutout( + w, config_provider("renewable", w.technology, "cutout")(w) + ), + output: + profile=resources("profile_pecd_{clusters}_{technology}.nc"), + log: + logs("build_renewable_profile_pecd_{clusters}_{technology}.log"), + benchmark: + benchmarks("build_renewable_profile_pecd_{clusters}_{technology}") + threads: 1 + resources: + mem_mb=4000, + wildcard_constraints: + technology="(?!hydro).*", # Any technology other than hydro + conda: + "../envs/environment.yaml" + script: + "../scripts/build_renewable_profiles_pecd.py" + + +rule clean_pecd_data: + params: + scenario=config_provider("tyndp_scenario"), + snapshots=config_provider("snapshots"), + input: + offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", + onshore_buses=resources("busmap_base_s_all.csv"), + fn_pecd="/Users/daniel/Desktop/Work/OET/Projects/open-tyndp/data/2024/20250313_ENTSO-E_ENTSOG_TYNDP_2024_Scenarios_Inputs/PECD/{planning_horizons}/", + output: + pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), + log: + logs("clean_pecd_data_{technology}_{planning_horizons}.log"), + benchmark: + benchmarks("clean_pecd_data_{technology}_{planning_horizons}") + threads: 4 + resources: + mem_mb=4000, + conda: + "../envs/environment.yaml" + script: + "../scripts/clean_pecd_data.py" + + rule build_monthly_prices: input: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py new file mode 100644 index 0000000000..3d28633ae8 --- /dev/null +++ b/scripts/build_renewable_profiles_pecd.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# +# SPDX-License-Identifier: MIT +""" +Creates renewable profiles for each region from PECD and TYNDP input data containing the available +generation time series (based on PECD weather data) from the node for onshore wind, AC-connected offshore wind, +DC-connected offshore wind and solar PV generators. + +.. note:: Hydroelectric profiles will be built in script :mod:`build_hydro_profiles_PECD`. Not yet implemented. + +Outputs +------- + +- ``resources/profile_pecd_{clusters}_{technology}.nc`` with the following structure + + =================== ==================== ========================================================= + Field Dimensions Description + =================== ==================== ========================================================= + profile year, bus, bin, time the per unit hourly availability factors for each bus + =================== ==================== ========================================================= +""" + +import logging + +import numpy as np +import pandas as pd +import xarray as xr + +from scripts._helpers import ( + configure_logging, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake( + "build_renewable_profiles_pecd", + clusters="all", + technology="offwind-ac", + ) + configure_logging(snakemake) + set_scenario_config(snakemake) + + technology = snakemake.wildcards.technology + pyears = snakemake.params.planning_horizons + + profiles = [] + + for year in pyears: + logger.info( + f"Extract PECD capacity factor time series for year {year} for technology {technology}..." + ) + + if int(year) not in [2030, 2040, 2050]: + year = np.clip(10 * (year // 10), 2030, 2050) + logger.warning( + "Planning horizon doesn't match available TYNDP data. " + f"Falling back to previous available year {year}." + ) + + profile = ( + pd.read_csv( + snakemake.input[f"pecd_data_{year}"], parse_dates=True, index_col=0 + ) + .rename_axis("time") + .reset_index() + .melt(id_vars=["time"], var_name="bus", value_name="profile") + .assign(bin=0, year=year) + .set_index(["time", "bus", "bin", "year"]) + .to_xarray() + ) + + profiles.append(profile) + + ds = xr.merge(profiles) + + # TODO: Later on the max capacities for renewable technologies and regions can be added here + # profiles = xr.merge(profiles) + # p_nom_max = place_holder + # average_distance = place_holder + # ds = xr.merge( + # [ + # profiles, + # p_nom_max.rename("p_nom_max"), + # average_distance.rename("average_distance"), + # ] + # ) + + ds.to_netcdf(snakemake.output.profile) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py new file mode 100644 index 0000000000..a14c5c3b45 --- /dev/null +++ b/scripts/clean_pecd_data.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# +# SPDX-License-Identifier: MIT +""" +Loads and cleans the available PECD capacity factor generation time series based on PECD weather data. +The script is executed for a given technology, and planning horizon. Technologies can be one of: + + * CSP_noStorage, + * CSP_withStorage, + * LFSolarPV, + * LFSolarPVRooftop, + * LFSolarPVUtility, + * Wind_Offshore, + * Wind_Onshore. + +Outputs +------- +Cleaned csv file with capacity factor generation time series and regions as columns. +""" + +import logging +import multiprocessing as mp +import os +from functools import partial +from pathlib import Path + +import pandas as pd +from tqdm import tqdm + +from scripts._helpers import ( + configure_logging, + get_snapshots, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + + +def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: str): + fn = Path(fn_pecd, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + + if not os.path.isfile(fn): + return None + + pecd_bus = pd.read_csv( + fn, + skiprows=10, + usecols=lambda name: name == "Date" or name == "Hour" or name == str(cyear), + ) + datetime_str = f"{cyear}." + pecd_bus["Date"].str.cat( + (pecd_bus["Hour"] - 1).astype(str), sep=" " + ) + cf_pecd = ( + pecd_bus.set_index(pd.to_datetime(datetime_str, format="%Y.%d.%m. %H")) + .drop(columns=["Date", "Hour"]) + .rename(columns={str(cyear): node}) + ) + + return cf_pecd + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake( + "clean_pecd_data", + clusters="all", + technology="offwind-ac", + planning_horizons=2030, + ) + configure_logging(snakemake) + set_scenario_config(snakemake) + + # Climate year from snapshots + cyear = get_snapshots(snakemake.params.snapshots)[0].year + if int(cyear) < 1982 or int(cyear) > 2019: + # TODO: Note that because of this fallback, the snapshots of the profiles will not always match with the model snapshots + logger.warning( + "Snapshot year doesn't match available TYNDP data. Falling back to 2009." + ) + cyear = 2009 + + # Planning year + pyear = str(snakemake.wildcards.planning_horizons) + + # Technology as in PECD terminology + # TODO: find solution for solar profiles being differentiated between Utility and Rooftop for Italy + pecd_tech_dict = { + "offwind-ac": "Wind_Offshore", + "offwind-dc": "Wind_Offshore", + "offwind-float": "Wind_Offshore", + "onwind": "Wind_Onshore", + "solar": "LFSolarPV", + "solar-hsat": "LFSolarPV", + } + pecd_tech = pecd_tech_dict[snakemake.wildcards.technology] + + offshore_buses = pd.read_excel(snakemake.input.offshore_buses, index_col=0) + onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) + + nodes = ( + offshore_buses.index if pecd_tech == "Wind_Offshore" else onshore_buses.index + ) + fn_pecd = snakemake.input.fn_pecd + + if pyear == "2050": + logger.warning( + "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." + ) + pyear = "2040" + fn_pecd = fn_pecd.replace("2050", "2040") + + # Load and prep electricity demand + tqdm_kwargs = { + "ascii": False, + "unit": " nodes", + "total": len(nodes), + "desc": "Loading PECD capacity factor data", + } + + func = partial( + read_pecd_file, + fn_pecd=fn_pecd, + cyear=cyear, + pyear=pyear, + technology=pecd_tech, + ) + + with mp.Pool(processes=snakemake.threads) as pool: + demand = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + + pecd_df = pd.concat(demand, axis=1) + + pecd_df.to_csv(snakemake.output.pecd_data_clean) From 9f0b674f88ef05e28d052e77d341fc0538ca6163 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 14:35:09 +0200 Subject: [PATCH 03/49] feat: add pecd profiles as input to prepare_sector_network and add_brownfield rules --- rules/build_sector.smk | 8 ++++++++ rules/solve_myopic.smk | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index b56071a4df..8088ee2262 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1230,6 +1230,13 @@ def input_profile_offwind(w): } +def input_profile_pecd(w): + return { + f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") + for tech in tyndp_renewable_profiles(w) + } + + rule build_egs_potentials: params: snapshots=config_provider("snapshots"), @@ -1380,6 +1387,7 @@ rule prepare_sector_network: scaling_factor=config_provider("load", "scaling_factor"), input: unpack(input_profile_offwind), + unpack(input_profile_pecd), unpack(input_heat_source_power), **rules.cluster_gas_network.output, **rules.build_gas_input_locations.output, diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index e2068bc817..a4c4dfdf9e 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -67,6 +67,13 @@ def input_profile_tech_brownfield(w): } +def input_profile_tech_brownfied_pecd(w): + return { + f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") + for tech in tyndp_renewable_profiles(w) + } + + rule add_brownfield: params: H2_retrofit=config_provider("sector", "H2_retrofit"), @@ -84,6 +91,7 @@ rule add_brownfield: ), input: unpack(input_profile_tech_brownfield), + unpack(input_profile_tech_brownfied_pecd), simplify_busmap=resources("busmap_base_s.csv"), cluster_busmap=resources("busmap_base_s_{clusters}.csv"), network=resources( From c0ce59fe7cf933fad5ff6ee150d2ef3401e14d8a Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 15:56:36 +0200 Subject: [PATCH 04/49] fix: add hotfix for add_existing_baseyear existing renewable capacities and fix name of profiles in add_brownfield rule --- rules/solve_myopic.smk | 3 ++- scripts/add_existing_baseyear.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index a4c4dfdf9e..7b6eb06a15 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -7,6 +7,7 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), + electricity=config_provider("electricity"), existing_capacities=config_provider("existing_capacities"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), @@ -69,7 +70,7 @@ def input_profile_tech_brownfield(w): def input_profile_tech_brownfied_pecd(w): return { - f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") + f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") for tech in tyndp_renewable_profiles(w) } diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index f49bd548b9..50d098133e 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -89,6 +89,13 @@ def add_existing_renewables( Modifies df_agg in-place """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} + # TODO: remove when TYNDP renewable generators are added + if len(tyndp_renewable_profiles) > 0: + logger.info( + f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'." + ) + for k in tyndp_renewable_profiles: + tech_map.pop(k, None) irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() irena = irena.query("Country in @countries") @@ -726,7 +733,11 @@ def add_heating_capacities_installed_before_baseyear( update_config_from_wildcards(snakemake.config, snakemake.wildcards) options = snakemake.params.sector - + tyndp_renewable_profiles = ( + snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] + if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] + else [] + ) baseyear = snakemake.params.baseyear n = pypsa.Network(snakemake.input.network) From cabc766afe04f4244162d33a94857de64eb20002 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 16:09:58 +0200 Subject: [PATCH 05/49] fix: fix pylint --- scripts/add_existing_baseyear.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 50d098133e..7c913b876c 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -90,6 +90,11 @@ def add_existing_renewables( """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} # TODO: remove when TYNDP renewable generators are added + tyndp_renewable_profiles = ( + snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] + if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] + else [] + ) if len(tyndp_renewable_profiles) > 0: logger.info( f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'." @@ -733,11 +738,7 @@ def add_heating_capacities_installed_before_baseyear( update_config_from_wildcards(snakemake.config, snakemake.wildcards) options = snakemake.params.sector - tyndp_renewable_profiles = ( - snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] - if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] - else [] - ) + baseyear = snakemake.params.baseyear n = pypsa.Network(snakemake.input.network) From 686df432e2a8c308fa16a9ce7b0111e927d68b25 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 16:20:14 +0200 Subject: [PATCH 06/49] fix: fix pylint better --- scripts/add_existing_baseyear.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 7c913b876c..d2bf9aac1b 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -68,6 +68,7 @@ def add_existing_renewables( costs: pd.DataFrame, df_agg: pd.DataFrame, countries: list[str], + tyndp_renewable_profiles: list[str], ) -> None: """ Add existing renewable capacities to conventional power plant data. @@ -82,6 +83,8 @@ def add_existing_renewables( Network containing topology and generator data countries : list List of country codes to consider + tyndp_renewable_profiles: list + List of renewable profile technologies taken from tyndp PECD Returns ------- @@ -90,11 +93,6 @@ def add_existing_renewables( """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} # TODO: remove when TYNDP renewable generators are added - tyndp_renewable_profiles = ( - snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] - if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] - else [] - ) if len(tyndp_renewable_profiles) > 0: logger.info( f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'." @@ -161,6 +159,7 @@ def add_power_capacities_installed_before_baseyear( countries: list[str], capacity_threshold: float, lifetime_values: dict[str, float], + tyndp_renewable_profiles: list[str], ) -> None: """ Add power generation capacities installed before base year. @@ -183,6 +182,8 @@ def add_power_capacities_installed_before_baseyear( Minimum capacity threshold lifetime_values : dict Default values for missing data + tyndp_renewable_profiles: list + List of renewable profile technologies taken from tyndp PECD """ logger.debug(f"Adding power capacities installed before {baseyear}") @@ -235,6 +236,7 @@ def add_power_capacities_installed_before_baseyear( costs=costs, n=n, countries=countries, + tyndp_renewable_profiles=tyndp_renewable_profiles, ) # drop assets which are already phased out / decommissioned phased_out = df_agg[df_agg["DateOut"] < baseyear].index @@ -739,6 +741,12 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector + tyndp_renewable_profiles = ( + snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] + if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] + else [] + ) + baseyear = snakemake.params.baseyear n = pypsa.Network(snakemake.input.network) @@ -765,6 +773,7 @@ def add_heating_capacities_installed_before_baseyear( countries=snakemake.config["countries"], capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"], lifetime_values=snakemake.params.costs["fill_values"], + tyndp_renewable_profiles=tyndp_renewable_profiles, ) if options["heating"]: From 4162538b129a343ee45d30d6fbe161afe0502346 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 17:58:32 +0200 Subject: [PATCH 07/49] fix: add electricity param for add_existing_baseyear rule in perfect foresight workflow --- rules/solve_perfect.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 8fb27f6ef2..3660dbf152 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -5,6 +5,7 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), + electricity=config_provider("electricity"), existing_capacities=config_provider("existing_capacities"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), From 79526a78cead02fa9e0d263d0b12a12b7e5e6fa3 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 13:02:51 +0200 Subject: [PATCH 08/49] feat: improve workflow implementation for more efficient execution of clean_pecd_data with handling of not available pecd years --- rules/build_electricity.smk | 12 ++++++------ scripts/build_renewable_profiles_pecd.py | 7 ++++++- scripts/build_tyndp_h2_network.py | 3 ++- scripts/clean_pecd_data.py | 7 ------- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 614cb0e112..ff056735b6 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -383,12 +383,12 @@ rule build_renewable_profiles: def input_data_pecd(w): return { - f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + pyear + ".csv") - for pyear in [ - "2030", - "2040", - "2050", - ] # complete PECD data is available for the years 2030, 2040 + f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv") + for pyear in set( + config_provider("scenario", "planning_horizons")(w) + ).intersection([2030, 2040]) + # Complete PECD data is only available for the years 2030, 2040 + # TODO: adjust if udpated 2050 data available } diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 3d28633ae8..b1dc15e62e 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -58,9 +58,14 @@ if int(year) not in [2030, 2040, 2050]: year = np.clip(10 * (year // 10), 2030, 2050) logger.warning( - "Planning horizon doesn't match available TYNDP data. " + "Planning horizon doesn't match available TYNDP PECD data. " f"Falling back to previous available year {year}." ) + if year == 2050: + logger.warning( + "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." + ) + year = 2040 profile = ( pd.read_csv( diff --git a/scripts/build_tyndp_h2_network.py b/scripts/build_tyndp_h2_network.py index 4aa7ec789d..86c5a2bf0d 100644 --- a/scripts/build_tyndp_h2_network.py +++ b/scripts/build_tyndp_h2_network.py @@ -10,6 +10,7 @@ import logging +import numpy as np import pandas as pd from _helpers import ( configure_logging, @@ -60,7 +61,7 @@ def load_h2_interzonal_connections(fn, scenario="GA", pyear=2030): "Planning horizon doesn't match available TYNDP data. " "Falling back to closest available year between 2030 and 2050." ) - pyear = min(max(2030, 5 * round(pyear / 5)), 2050) + pyear = np.clip(5 * (pyear // 5), 2030, 2050) scenario_dict = { "GA": "Global Ambition", "DE": "Distributed Energy", diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index a14c5c3b45..89bbf78248 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -104,13 +104,6 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: ) fn_pecd = snakemake.input.fn_pecd - if pyear == "2050": - logger.warning( - "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." - ) - pyear = "2040" - fn_pecd = fn_pecd.replace("2050", "2040") - # Load and prep electricity demand tqdm_kwargs = { "ascii": False, From ba5458da68fd24ada97ccb7000a0ba27a5b5dcf9 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 14:09:41 +0200 Subject: [PATCH 09/49] fix: hotfix for add_brownfield p_max_pu until tyndp generators are added. Needs to be reverted later --- rules/solve_myopic.smk | 1 + scripts/add_brownfield.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 7b6eb06a15..692785fa7c 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -83,6 +83,7 @@ rule add_brownfield: ), threshold_capacity=config_provider("existing_capacities", "threshold_capacity"), snapshots=config_provider("snapshots"), + electricity=config_provider("electricity"), drop_leap_day=config_provider("enable", "drop_leap_day"), carriers=config_provider("electricity", "renewable_carriers"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1e92d6558c..96bb1a4e9b 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -218,7 +218,10 @@ def adjust_renewable_profiles(n, input_profiles, params, year): pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill() ) - for carrier in params["carriers"]: + # TODO: hotfix remove filter for tyndp_renewable_profiles after tyndp generators are added + for carrier in set(params["carriers"]) - set( + params["electricity"]["tyndp_renewable_profiles"]["technologies"] + ): if carrier == "hydro": continue From 81402fe4bd47827f5bbb95ee38132115f8d1316e Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 14:13:26 +0200 Subject: [PATCH 10/49] doc: add description of new configuration option to configtables --- doc/configtables/electricity.csv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 4ea09ef663..0a738bc50e 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -30,6 +30,9 @@ everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignit conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound." ,,, renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. +tyndp_renewable_profiles,,, +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. +-- technologies,--,list,Select the renewable technologies for which PECD renewable profiles are used. Carriers not listed use default renewable profiles. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. From c1fbe1392b26cefe4bb9ee1099c648617f06ceed Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 14:17:50 +0200 Subject: [PATCH 11/49] doc: add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index df7a923699..15e69b7671 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,6 +13,8 @@ Release Notes **Changes** +* Add processing and preparation of TYNDP 2024 PECD renewable profiles instead of default renewable profiles using atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. + * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards. * Add the TYNDP electricity demand as an exogenously set demand (https://github.com/open-energy-transition/open-tyndp/pull/14). This requires the default PyPSA-Eur modelling to be explicitly disabled. The TYNDP electricity demand depends on the planning year, necessitating a different approach to the default PyPSA-Eur one. Wildcards are introduced and load is attached in `prepare_sector_network`. From b684567d5bb8e43939ce6be07edb28e38899fcf8 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 16:01:29 +0200 Subject: [PATCH 12/49] feat: add retrieval rule for PECD data from google drive until data on Zenodo --- rules/build_electricity.smk | 5 ++- rules/retrieve.smk | 14 ++++++++ scripts/clean_pecd_data.py | 2 +- scripts/retrieve_tyndp_pecd_data.py | 56 +++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 scripts/retrieve_tyndp_pecd_data.py diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index ff056735b6..e3e596bdb6 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -435,7 +435,7 @@ rule clean_pecd_data: input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), - fn_pecd="/Users/daniel/Desktop/Work/OET/Projects/open-tyndp/data/2024/20250313_ENTSO-E_ENTSOG_TYNDP_2024_Scenarios_Inputs/PECD/{planning_horizons}/", + fn_pecd="data/tyndp_2024_bundle/PECD", output: pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), log: @@ -451,6 +451,9 @@ rule clean_pecd_data: "../scripts/clean_pecd_data.py" +ruleorder: retrieve_tyndp_pecd_data > clean_pecd_data + + rule build_monthly_prices: input: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4f2b4646de..a50c261347 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -185,6 +185,20 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" script: "../scripts/retrieve_tyndp_bundle.py" + rule retrieve_tyndp_pecd_data: + params: + tyndp_bundle="data/tyndp_2024_bundle", + # TODO Integrate into Zenodo tyndp data bundle + output: + dir=directory("data/tyndp_2024_bundle/PECD"), + log: + "logs/retrieve_tyndp_pecd_data.log", + retries: 2 + script: + "../scripts/retrieve_tyndp_pecd_data.py" + + ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data + rule retrieve_countries_centroids: output: "data/countries_centroids.geojson", diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 89bbf78248..78f126b8df 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -37,7 +37,7 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: str): - fn = Path(fn_pecd, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + fn = Path(fn_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") if not os.path.isfile(fn): return None diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py new file mode 100644 index 0000000000..3a780113ca --- /dev/null +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: 2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +The TYNDP PECD data contains input data for the 2024 TYNDP scenario building process. + +This rule downloads the TYNDP PECD data from `a gdrive and extracts it in the ``data/tyndp_2024_bundle`` +subdirectory, such that all files of the TYNDP bundle are stored in it. + +**Outputs** + +- ``data/tyndp_2024_bundle/PECD``: PECD input data for TYNDP 2024 scenario building + +""" + +import logging +import os +import zipfile + +from _helpers import configure_logging, progress_retrieve, set_scenario_config + +logger = logging.getLogger(__name__) + +# Define the base URL +url = "https://drive.usercontent.google.com/download?id=1kgCQON-XXxRgDL7gX_uMulCV25RCFr4F&export=download&authuser=0&confirm=t" + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_tyndp_bundle") + rootpath = ".." + else: + rootpath = "." + + configure_logging(snakemake) + set_scenario_config(snakemake) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + + to_fn = snakemake.output.dir + tyndp_bundle_fn = snakemake.params["tyndp_bundle"] + to_fn_zp = to_fn + ".zip" + + # download .zip file + logger.info(f"Downloading TYNDP PECD data from '{url}'.") + progress_retrieve(url, to_fn_zp, disable=disable_progress) + + # extract + logger.info("Extracting TYNDP PECD data.") + with zipfile.ZipFile(to_fn_zp, "r") as zip_ref: + zip_ref.extractall(tyndp_bundle_fn) + + # remove .zip file + os.remove(to_fn_zp) + + logger.info(f"TYNDP PECD data available in '{to_fn}'.") From c33f5a332bb86fdd4924cd3dda0df41259e9a152 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 16:17:08 +0200 Subject: [PATCH 13/49] fix: fix ruleorder statement --- rules/build_electricity.smk | 3 --- rules/retrieve.smk | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index e3e596bdb6..4b7886e253 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -451,9 +451,6 @@ rule clean_pecd_data: "../scripts/clean_pecd_data.py" -ruleorder: retrieve_tyndp_pecd_data > clean_pecd_data - - rule build_monthly_prices: input: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index a50c261347..724b248974 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -197,7 +197,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" script: "../scripts/retrieve_tyndp_pecd_data.py" - ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data + ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data rule retrieve_countries_centroids: output: From 16f9c160347e185591b756e04e722e8ccaeb8d1e Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 17:33:15 +0200 Subject: [PATCH 14/49] fix: remove unneeded inputs to build_renewable_profiles_pecd and update download link for pecd data --- rules/build_electricity.smk | 11 ----------- scripts/retrieve_tyndp_pecd_data.py | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 4b7886e253..8c2bba23cd 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -400,17 +400,6 @@ rule build_renewable_profiles_pecd: planning_horizons=config_provider("scenario", "planning_horizons"), input: unpack(input_data_pecd), - availability_matrix=resources("availability_matrix_{clusters}_{technology}.nc"), - offshore_shapes=resources("offshore_shapes.geojson"), - distance_regions=resources("regions_onshore_base_s_{clusters}.geojson"), - resource_regions=lambda w: ( - resources("regions_onshore_base_s_{clusters}.geojson") - if w.technology in ("onwind", "solar", "solar-hsat") - else resources("regions_offshore_base_s_{clusters}.geojson") - ), - cutout=lambda w: input_cutout( - w, config_provider("renewable", w.technology, "cutout")(w) - ), output: profile=resources("profile_pecd_{clusters}_{technology}.nc"), log: diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 3a780113ca..528111ab2e 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) # Define the base URL -url = "https://drive.usercontent.google.com/download?id=1kgCQON-XXxRgDL7gX_uMulCV25RCFr4F&export=download&authuser=0&confirm=t" +url = "https://drive.usercontent.google.com/download?id=15YF5_DIhIKrkJvhwbzbj3FKTyTzrtfda&export=download&authuser=0&confirm=t" if __name__ == "__main__": if "snakemake" not in globals(): From d5bc7eae6c6e21e0fe8de90cdf7632f60cc79259 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 2 Jun 2025 15:43:35 +0200 Subject: [PATCH 15/49] doc: update license identifier --- scripts/add_brownfield.py | 2 +- scripts/build_renewable_profiles_pecd.py | 2 +- scripts/clean_pecd_data.py | 2 +- scripts/retrieve_tyndp_pecd_data.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 96bb1a4e9b..cf4eb621bf 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH and contributors to PyPSA-Eur # # SPDX-License-Identifier: MIT """ diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index b1dc15e62e..23cee8cd9a 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH # # SPDX-License-Identifier: MIT """ diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 78f126b8df..a41474bd8d 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH # # SPDX-License-Identifier: MIT """ diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 528111ab2e..47d87a9539 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2024 The PyPSA-Eur Authors +# SPDX-FileCopyrightText: Open Energy Transition gGmbH # # SPDX-License-Identifier: MIT """ From 500082ccd414e4a1e31ceb301c6f133719709b20 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 3 Jun 2025 12:17:07 +0200 Subject: [PATCH 16/49] feat: update pecd retrieval to use gcp storage --- scripts/retrieve_tyndp_pecd_data.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 47d87a9539..a24bbc3675 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -16,19 +16,20 @@ import logging import os import zipfile +from pathlib import Path from _helpers import configure_logging, progress_retrieve, set_scenario_config logger = logging.getLogger(__name__) # Define the base URL -url = "https://drive.usercontent.google.com/download?id=15YF5_DIhIKrkJvhwbzbj3FKTyTzrtfda&export=download&authuser=0&confirm=t" +url = "https://storage.googleapis.com/open-tyndp-data-store/PECD.zip" if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("retrieve_tyndp_bundle") + snakemake = mock_snakemake("retrieve_tyndp_pecd_data") rootpath = ".." else: rootpath = "." @@ -38,7 +39,7 @@ disable_progress = snakemake.config["run"].get("disable_progressbar", False) to_fn = snakemake.output.dir - tyndp_bundle_fn = snakemake.params["tyndp_bundle"] + tyndp_bundle_fn = Path(rootpath, snakemake.params["tyndp_bundle"]) to_fn_zp = to_fn + ".zip" # download .zip file From 4cdfe9dd50511ba97bddd93d121ee21aada694c6 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 12 Jun 2025 17:33:38 +0200 Subject: [PATCH 17/49] refactor: use tyndp specific renewable carriers and differentiate between tyndp renewable carriers and pecd renewable profiles --- config/config.default.yaml | 14 +++++--- config/config.tyndp.yaml | 16 ++++++--- config/test/config.tyndp.yaml | 16 ++++++--- doc/configtables/electricity.csv | 5 +-- rules/build_electricity.smk | 62 +++++++++++++++++--------------- rules/build_sector.smk | 17 +++++++-- rules/solve_myopic.smk | 4 +-- scripts/add_brownfield.py | 21 ++++++++--- scripts/add_electricity.py | 20 +++++++---- scripts/add_existing_baseyear.py | 34 ++++++++++-------- scripts/clean_pecd_data.py | 1 + 11 files changed, 138 insertions(+), 72 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 234b0ecabc..23c5ddfa14 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -111,12 +111,18 @@ electricity: conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float, hydro] - tyndp_renewable_profiles: + pecd_renewable_profiles: enable: false technologies: - - offwind-ac - - offwind-dc - - offwind-float + offwind: + - offwind-ac-fb-r + - offwind-ac-fl-r + - offwind-dc-fb-r + - offwind-dc-fl-r + - offwind-dc-fb-oh + - offwind-dc-fl-oh + - offwind-h2-fb-oh + - offwind-h2-fl-oh estimate_renewable_capacities: enable: true diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 8a0836ec6c..6f6514d272 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -38,12 +38,20 @@ co2_budget: electricity: base_network: tyndp-raw transmission_limit: v1.0 - tyndp_renewable_profiles: + pecd_renewable_profiles: enable: true technologies: - - offwind-ac - - offwind-dc - - offwind-float + offwind: + - offwind-ac-fb-r + - offwind-ac-fl-r + - offwind-dc-fb-r + - offwind-dc-fl-r + - offwind-dc-fb-oh + - offwind-dc-fl-oh + - offwind-h2-fb-oh + - offwind-h2-fl-oh + + renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro] links: p_max_pu: 1.0 diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 4cbd874a44..797edd2c3c 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -35,12 +35,18 @@ co2_budget: electricity: base_network: tyndp-raw transmission_limit: v1.0 - tyndp_renewable_profiles: + pecd_renewable_profiles: enable: true technologies: - - offwind-ac - - offwind-dc - - offwind-float + offwind: + - offwind-ac-fb-r + - offwind-ac-fl-r + - offwind-dc-fb-r + - offwind-dc-fl-r + - offwind-dc-fb-oh + - offwind-dc-fl-oh + - offwind-h2-fb-oh + - offwind-h2-fl-oh extendable_carriers: Generator: [OCGT] @@ -48,7 +54,7 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float] + renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro] atlite: default_cutout: europe-2013-03-sarah3-era5 diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 0a738bc50e..3af8782456 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -30,9 +30,10 @@ everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignit conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound." ,,, renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. -tyndp_renewable_profiles,,, +pecd_renewable_profiles,,, -- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. --- technologies,--,list,Select the renewable technologies for which PECD renewable profiles are used. Carriers not listed use default renewable profiles. +-- technologies,,, +-- -- {pecd_profile},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. Carriers not listed use default renewable profiles. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 8c2bba23cd..9c342da7ad 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -381,6 +381,29 @@ rule build_renewable_profiles: "../scripts/build_renewable_profiles.py" +rule clean_pecd_data: + params: + scenario=config_provider("tyndp_scenario"), + snapshots=config_provider("snapshots"), + input: + offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", + onshore_buses=resources("busmap_base_s_all.csv"), + fn_pecd="data/tyndp_2024_bundle/PECD", + output: + pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), + log: + logs("clean_pecd_data_{technology}_{planning_horizons}.log"), + benchmark: + benchmarks("clean_pecd_data_{technology}_{planning_horizons}") + threads: 4 + resources: + mem_mb=4000, + conda: + "../envs/environment.yaml" + script: + "../scripts/clean_pecd_data.py" + + def input_data_pecd(w): return { f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv") @@ -417,29 +440,6 @@ rule build_renewable_profiles_pecd: "../scripts/build_renewable_profiles_pecd.py" -rule clean_pecd_data: - params: - scenario=config_provider("tyndp_scenario"), - snapshots=config_provider("snapshots"), - input: - offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", - onshore_buses=resources("busmap_base_s_all.csv"), - fn_pecd="data/tyndp_2024_bundle/PECD", - output: - pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), - log: - logs("clean_pecd_data_{technology}_{planning_horizons}.log"), - benchmark: - benchmarks("clean_pecd_data_{technology}_{planning_horizons}") - threads: 4 - resources: - mem_mb=4000, - conda: - "../envs/environment.yaml" - script: - "../scripts/clean_pecd_data.py" - - rule build_monthly_prices: input: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", @@ -599,7 +599,7 @@ def input_class_regions(w): ) for tech in set(config_provider("electricity", "renewable_carriers")(w)) - {"hydro"} - - set(tyndp_renewable_profiles(w)) + - set(tyndp_renewable_carriers(w)) } @@ -765,10 +765,16 @@ rule cluster_network: "../scripts/cluster_network.py" -def tyndp_renewable_profiles(w): +def tyndp_renewable_carriers(w): return ( - config_provider("electricity", "tyndp_renewable_profiles", "technologies")(w) - if config_provider("electricity", "tyndp_renewable_profiles", "enable")(w) + [ + subcarrier + for carrier in config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w).values() + for subcarrier in carrier + ] + if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) else [] ) @@ -781,7 +787,7 @@ def input_profile_tech(w): else f"profile_{tech}.nc" ) for tech in set(config_provider("electricity", "renewable_carriers")(w)) - - set(tyndp_renewable_profiles(w)) + - set(tyndp_renewable_carriers(w)) } diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 8088ee2262..9853ba5ccf 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1226,14 +1226,27 @@ def input_profile_offwind(w): f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") for tech in ["offwind-ac", "offwind-dc", "offwind-float"] if (tech in config_provider("electricity", "renewable_carriers")(w)) - and (tech not in tyndp_renewable_profiles(w)) + and (tech not in tyndp_renewable_carriers(w)) } +def pecd_renewable_profiles(w): + return ( + [ + carrier + for carrier in config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w).keys() + ] + if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) + else [] + ) + + def input_profile_pecd(w): return { f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") - for tech in tyndp_renewable_profiles(w) + for tech in pecd_renewable_profiles(w) } diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 692785fa7c..ac721e8d2c 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -62,7 +62,7 @@ def input_profile_tech_brownfield(w): f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") for tech in ( set(config_provider("electricity", "renewable_carriers")(w)) - - set(tyndp_renewable_profiles(w)) + - set(tyndp_renewable_carriers(w)) ) if tech != "hydro" } @@ -71,7 +71,7 @@ def input_profile_tech_brownfield(w): def input_profile_tech_brownfied_pecd(w): return { f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") - for tech in tyndp_renewable_profiles(w) + for tech in pecd_renewable_profiles(w) } diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index cf4eb621bf..4c8512980a 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -218,10 +218,23 @@ def adjust_renewable_profiles(n, input_profiles, params, year): pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill() ) - # TODO: hotfix remove filter for tyndp_renewable_profiles after tyndp generators are added - for carrier in set(params["carriers"]) - set( - params["electricity"]["tyndp_renewable_profiles"]["technologies"] - ): + # TODO: hotfix remove filter for tyndp_renewable_carriers after tyndp generators are added + tyndp_renewable_carriers = ( + [ + subcarrier + for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ + "technologies" + ].values() + for subcarrier in carrier + ] + if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] + else [] + ) + if len(tyndp_renewable_carriers) > 0: + logger.info( + f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." + ) + for carrier in set(params["carriers"]) - set(tyndp_renewable_carriers): if carrier == "hydro": continue diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index d7c9a314cb..176f6c0a1c 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1203,20 +1203,26 @@ def attach_stores( params.link_length_factor, ) - tyndp_renewable_profiles = ( - params.electricity["tyndp_renewable_profiles"]["technologies"] - if params.electricity["tyndp_renewable_profiles"]["enable"] + tyndp_renewable_carriers = ( + [ + subcarrier + for carrier in params.electricity["pecd_renewable_profiles"][ + "technologies" + ].values() + for subcarrier in carrier + ] + if params.electricity["pecd_renewable_profiles"]["enable"] else [] ) - if len(tyndp_renewable_profiles) > 0: + if len(tyndp_renewable_carriers) > 0: logger.info( - f"Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'. They will be attached later on with TYNDP data." + f"Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'. They will be attached later on with TYNDP data." ) renewable_carriers = set( [ carrier for carrier in params.electricity["renewable_carriers"] - if carrier not in tyndp_renewable_profiles + if carrier not in tyndp_renewable_carriers ] ) extendable_carriers = params.electricity["extendable_carriers"] @@ -1285,7 +1291,7 @@ def attach_stores( tech_map = { key: value for key, value in estimate_renewable_caps["technology_mapping"].items() - if value not in tyndp_renewable_profiles + if value not in tyndp_renewable_carriers } expansion_limit = estimate_renewable_caps["expansion_limit"] year = estimate_renewable_caps["year"] diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index d2bf9aac1b..d14ad9a850 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -68,7 +68,7 @@ def add_existing_renewables( costs: pd.DataFrame, df_agg: pd.DataFrame, countries: list[str], - tyndp_renewable_profiles: list[str], + tyndp_renewable_carriers: list[str], ) -> None: """ Add existing renewable capacities to conventional power plant data. @@ -83,8 +83,8 @@ def add_existing_renewables( Network containing topology and generator data countries : list List of country codes to consider - tyndp_renewable_profiles: list - List of renewable profile technologies taken from tyndp PECD + tyndp_renewable_carriers: list + List of renewable technologies from TYNDP Returns ------- @@ -93,11 +93,11 @@ def add_existing_renewables( """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} # TODO: remove when TYNDP renewable generators are added - if len(tyndp_renewable_profiles) > 0: + if len(tyndp_renewable_carriers) > 0: logger.info( - f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'." + f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." ) - for k in tyndp_renewable_profiles: + for k in tyndp_renewable_carriers: tech_map.pop(k, None) irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() @@ -159,7 +159,7 @@ def add_power_capacities_installed_before_baseyear( countries: list[str], capacity_threshold: float, lifetime_values: dict[str, float], - tyndp_renewable_profiles: list[str], + tyndp_renewable_carriers: list[str], ) -> None: """ Add power generation capacities installed before base year. @@ -182,8 +182,8 @@ def add_power_capacities_installed_before_baseyear( Minimum capacity threshold lifetime_values : dict Default values for missing data - tyndp_renewable_profiles: list - List of renewable profile technologies taken from tyndp PECD + tyndp_renewable_carriers: list + List of renewable technologies from TYNDP """ logger.debug(f"Adding power capacities installed before {baseyear}") @@ -236,7 +236,7 @@ def add_power_capacities_installed_before_baseyear( costs=costs, n=n, countries=countries, - tyndp_renewable_profiles=tyndp_renewable_profiles, + tyndp_renewable_carriers=tyndp_renewable_carriers, ) # drop assets which are already phased out / decommissioned phased_out = df_agg[df_agg["DateOut"] < baseyear].index @@ -741,9 +741,15 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector - tyndp_renewable_profiles = ( - snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] - if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] + tyndp_renewable_carriers = ( + [ + subcarrier + for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ + "technologies" + ].values() + for subcarrier in carrier + ] + if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] else [] ) @@ -773,7 +779,7 @@ def add_heating_capacities_installed_before_baseyear( countries=snakemake.config["countries"], capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"], lifetime_values=snakemake.params.costs["fill_values"], - tyndp_renewable_profiles=tyndp_renewable_profiles, + tyndp_renewable_carriers=tyndp_renewable_carriers, ) if options["heating"]: diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index a41474bd8d..65f5a8994b 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -90,6 +90,7 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: "offwind-ac": "Wind_Offshore", "offwind-dc": "Wind_Offshore", "offwind-float": "Wind_Offshore", + "offwind": "Wind_Offshore", "onwind": "Wind_Onshore", "solar": "LFSolarPV", "solar-hsat": "LFSolarPV", From f2af81bbdf0e9e2b7f4b39a801674be914d98c8e Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 12 Jun 2025 17:58:16 +0200 Subject: [PATCH 18/49] bugfix: include missing offshore node PECD data as empty columns and replace UK with GB naming convention --- scripts/clean_pecd_data.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 65f5a8994b..f7a2928f66 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -124,6 +124,13 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: with mp.Pool(processes=snakemake.threads) as pool: demand = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) - pecd_df = pd.concat(demand, axis=1) + pecd_df = ( + pd.concat(demand, axis=1) + .reindex(nodes, axis=1) + .fillna(0.0) # include missing node data with empty columns + .rename( + columns=lambda x: x.replace("UK", "GB") + ) # replace UK with GB for naming convention + ) pecd_df.to_csv(snakemake.output.pecd_data_clean) From cf11da32476c428bef8f0d592f83472ee34a3f93 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 13 Jun 2025 11:28:40 +0200 Subject: [PATCH 19/49] bugfix: fix pylint and remove hydro from test config --- config/test/config.tyndp.yaml | 2 +- scripts/add_brownfield.py | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index a363a06c6a..38a12ae62a 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -56,7 +56,7 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro] + renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] atlite: default_cutout: europe-2013-03-sarah3-era5 diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 4c8512980a..127c24d0bd 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -206,7 +206,9 @@ def disable_grid_expansion_if_limit_hit(n): n.global_constraints.drop(name, inplace=True) -def adjust_renewable_profiles(n, input_profiles, params, year): +def adjust_renewable_profiles( + n, input_profiles, params, year, tyndp_renewable_carriers +): """ Adjusts renewable profiles according to the renewable technology specified, using the latest year below or equal to the selected year. @@ -219,17 +221,6 @@ def adjust_renewable_profiles(n, input_profiles, params, year): ) # TODO: hotfix remove filter for tyndp_renewable_carriers after tyndp generators are added - tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] - if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] - else [] - ) if len(tyndp_renewable_carriers) > 0: logger.info( f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." @@ -365,7 +356,21 @@ def update_dynamic_ptes_capacity( n = pypsa.Network(snakemake.input.network) - adjust_renewable_profiles(n, snakemake.input, snakemake.params, year) + tyndp_renewable_carriers = ( + [ + subcarrier + for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ + "technologies" + ].values() + for subcarrier in carrier + ] + if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] + else [] + ) + + adjust_renewable_profiles( + n, snakemake.input, snakemake.params, year, tyndp_renewable_carriers + ) add_build_year_to_new_assets(n, year) From d84159ffcbb2d58d8ae2f03c7304239d627f29a6 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 13 Jun 2025 11:30:39 +0200 Subject: [PATCH 20/49] bugfix: add existing res dependent on renewable carriers list --- rules/solve_myopic.smk | 1 + scripts/add_existing_baseyear.py | 91 ++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 41 deletions(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index ac721e8d2c..dce9c4e3ad 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -9,6 +9,7 @@ rule add_existing_baseyear: sector=config_provider("sector"), electricity=config_provider("electricity"), existing_capacities=config_provider("existing_capacities"), + carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), energy_totals_year=config_provider("energy", "energy_totals_year"), diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index d14ad9a850..8609e02581 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -68,6 +68,7 @@ def add_existing_renewables( costs: pd.DataFrame, df_agg: pd.DataFrame, countries: list[str], + renewable_carriers: list[str], tyndp_renewable_carriers: list[str], ) -> None: """ @@ -83,6 +84,8 @@ def add_existing_renewables( Network containing topology and generator data countries : list List of country codes to consider + renewable_carriers: list + List of renewable carriers in the network tyndp_renewable_carriers: list List of renewable technologies from TYNDP @@ -97,8 +100,7 @@ def add_existing_renewables( logger.info( f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." ) - for k in tyndp_renewable_carriers: - tech_map.pop(k, None) + renewable_carriers = set(renewable_carriers) - set(tyndp_renewable_carriers) irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() irena = irena.query("Country in @countries") @@ -107,45 +109,46 @@ def add_existing_renewables( irena = irena.unstack().reset_index() for carrier, tech in tech_map.items(): - df = ( - irena[irena.Technology.str.contains(tech)] - .drop(columns=["Technology"]) - .set_index("Country") - ) - df.columns = df.columns.astype(int) - - # calculate yearly differences - df.insert(loc=0, value=0.0, column="1999") - df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0) - - # distribute capacities among generators potential (p_nom_max) - gen_i = n.generators.query("carrier == @carrier").index - carrier_gens = n.generators.loc[gen_i] - res_capacities = [] - for country, group in carrier_gens.groupby( - carrier_gens.bus.map(n.buses.country) - ): - fraction = group.p_nom_max / group.p_nom_max.sum() - res_capacities.append(cartesian(df.loc[country], fraction)) - res_capacities = pd.concat(res_capacities, axis=1).T - - for year in res_capacities.columns: - for gen in res_capacities.index: - bus_bin = re.sub(f" {carrier}.*", "", gen) - bus, bin_id = bus_bin.rsplit(" ", maxsplit=1) - name = f"{bus_bin} {carrier}-{year}" - capacity = res_capacities.loc[gen, year] - if capacity > 0.0: - cost_key = carrier.split("-", maxsplit=1)[0] - df_agg.at[name, "Fueltype"] = carrier - df_agg.at[name, "Capacity"] = capacity - df_agg.at[name, "DateIn"] = year - df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"] - df_agg.at[name, "DateOut"] = ( - year + costs.at[cost_key, "lifetime"] - 1 - ) - df_agg.at[name, "bus"] = bus - df_agg.at[name, "resource_class"] = bin_id + if carrier in renewable_carriers: + df = ( + irena[irena.Technology.str.contains(tech)] + .drop(columns=["Technology"]) + .set_index("Country") + ) + df.columns = df.columns.astype(int) + + # calculate yearly differences + df.insert(loc=0, value=0.0, column="1999") + df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0) + + # distribute capacities among generators potential (p_nom_max) + gen_i = n.generators.query("carrier == @carrier").index + carrier_gens = n.generators.loc[gen_i] + res_capacities = [] + for country, group in carrier_gens.groupby( + carrier_gens.bus.map(n.buses.country) + ): + fraction = group.p_nom_max / group.p_nom_max.sum() + res_capacities.append(cartesian(df.loc[country], fraction)) + res_capacities = pd.concat(res_capacities, axis=1).T + + for year in res_capacities.columns: + for gen in res_capacities.index: + bus_bin = re.sub(f" {carrier}.*", "", gen) + bus, bin_id = bus_bin.rsplit(" ", maxsplit=1) + name = f"{bus_bin} {carrier}-{year}" + capacity = res_capacities.loc[gen, year] + if capacity > 0.0: + cost_key = carrier.split("-", maxsplit=1)[0] + df_agg.at[name, "Fueltype"] = carrier + df_agg.at[name, "Capacity"] = capacity + df_agg.at[name, "DateIn"] = year + df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"] + df_agg.at[name, "DateOut"] = ( + year + costs.at[cost_key, "lifetime"] - 1 + ) + df_agg.at[name, "bus"] = bus + df_agg.at[name, "resource_class"] = bin_id df_agg["resource_class"] = df_agg["resource_class"].fillna(0) @@ -159,6 +162,7 @@ def add_power_capacities_installed_before_baseyear( countries: list[str], capacity_threshold: float, lifetime_values: dict[str, float], + renewable_carriers: list[str], tyndp_renewable_carriers: list[str], ) -> None: """ @@ -182,6 +186,8 @@ def add_power_capacities_installed_before_baseyear( Minimum capacity threshold lifetime_values : dict Default values for missing data + renewable_carriers: list + List of renewable carriers in the network tyndp_renewable_carriers: list List of renewable technologies from TYNDP """ @@ -236,6 +242,7 @@ def add_power_capacities_installed_before_baseyear( costs=costs, n=n, countries=countries, + renewable_carriers=renewable_carriers, tyndp_renewable_carriers=tyndp_renewable_carriers, ) # drop assets which are already phased out / decommissioned @@ -741,6 +748,7 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector + renewable_carriers = snakemake.params.carriers tyndp_renewable_carriers = ( [ subcarrier @@ -779,6 +787,7 @@ def add_heating_capacities_installed_before_baseyear( countries=snakemake.config["countries"], capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"], lifetime_values=snakemake.params.costs["fill_values"], + renewable_carriers=renewable_carriers, tyndp_renewable_carriers=tyndp_renewable_carriers, ) From e0635cbb9f5337e47c8160ab71ab4ef1a5bd10d5 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 13 Jun 2025 11:32:12 +0200 Subject: [PATCH 21/49] feature: add 2050 pecd renewable profiles explicitly and filter for snapshots when cleaning data --- rules/build_electricity.smk | 1 + scripts/build_renewable_profiles_pecd.py | 3 ++- scripts/clean_pecd_data.py | 22 ++++++++++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index c36e9e774e..8513eb5694 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -397,6 +397,7 @@ rule clean_pecd_data: params: scenario=config_provider("tyndp_scenario"), snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 23cee8cd9a..c1793876cd 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -61,6 +61,7 @@ "Planning horizon doesn't match available TYNDP PECD data. " f"Falling back to previous available year {year}." ) + year_i = str(year) if year == 2050: logger.warning( "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." @@ -74,7 +75,7 @@ .rename_axis("time") .reset_index() .melt(id_vars=["time"], var_name="bus", value_name="profile") - .assign(bin=0, year=year) + .assign(bin=0, year=year_i) .set_index(["time", "bus", "bin", "year"]) .to_xarray() ) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index f7a2928f66..98f6826dec 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -36,7 +36,14 @@ logger = logging.getLogger(__name__) -def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: str): +def read_pecd_file( + node: list, + fn_pecd: str, + cyear: str, + pyear: str, + technology: str, + sns: pd.DatetimeIndex, +): fn = Path(fn_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") if not os.path.isfile(fn): @@ -45,8 +52,12 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: pecd_bus = pd.read_csv( fn, skiprows=10, - usecols=lambda name: name == "Date" or name == "Hour" or name == str(cyear), - ) + usecols=lambda name: name == "Date" + or name == "Hour" + or name == str(cyear) + or name == str(float(cyear)), + ).rename(columns={str(float(cyear)): str(cyear)}) + datetime_str = f"{cyear}." + pecd_bus["Date"].str.cat( (pecd_bus["Hour"] - 1).astype(str), sep=" " ) @@ -54,6 +65,7 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: pecd_bus.set_index(pd.to_datetime(datetime_str, format="%Y.%d.%m. %H")) .drop(columns=["Date", "Hour"]) .rename(columns={str(cyear): node}) + .loc[sns] # filter for snapshots only ) return cf_pecd @@ -73,7 +85,8 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: set_scenario_config(snakemake) # Climate year from snapshots - cyear = get_snapshots(snakemake.params.snapshots)[0].year + sns = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + cyear = sns[0].year if int(cyear) < 1982 or int(cyear) > 2019: # TODO: Note that because of this fallback, the snapshots of the profiles will not always match with the model snapshots logger.warning( @@ -119,6 +132,7 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: cyear=cyear, pyear=pyear, technology=pecd_tech, + sns=sns, ) with mp.Pool(processes=snakemake.threads) as pool: From 7b297a36adba7a480c4656e7882ca9cfe589ff89 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 13 Jun 2025 13:08:00 +0200 Subject: [PATCH 22/49] bugfix: fix params for add_existing_baseyear solve_perfect --- rules/solve_perfect.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 3660dbf152..545b547705 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -7,6 +7,7 @@ rule add_existing_baseyear: sector=config_provider("sector"), electricity=config_provider("electricity"), existing_capacities=config_provider("existing_capacities"), + carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), energy_totals_year=config_provider("energy", "energy_totals_year"), From fe8cb522037efa0e083331a17a51e8d40ff66cac Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 16 Jun 2025 11:55:42 +0200 Subject: [PATCH 23/49] bugfix: change year in renewable profile pecd back to integer --- scripts/build_renewable_profiles_pecd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index c1793876cd..68fa9acf85 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -61,7 +61,7 @@ "Planning horizon doesn't match available TYNDP PECD data. " f"Falling back to previous available year {year}." ) - year_i = str(year) + year_i = year if year == 2050: logger.warning( "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." From 67c84ced3e8dcb69d0b06567766b8fd862baa06c Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 19 Jun 2025 13:09:37 +0200 Subject: [PATCH 24/49] bugfix: fix year dimension for 2035 an 2045 steps --- scripts/build_renewable_profiles_pecd.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 68fa9acf85..39197fb9c5 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -54,14 +54,13 @@ logger.info( f"Extract PECD capacity factor time series for year {year} for technology {technology}..." ) - + year_i = year if int(year) not in [2030, 2040, 2050]: year = np.clip(10 * (year // 10), 2030, 2050) logger.warning( "Planning horizon doesn't match available TYNDP PECD data. " f"Falling back to previous available year {year}." ) - year_i = year if year == 2050: logger.warning( "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." From 225de715001262585dbeec57e419766d441d359e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20R=C3=BCdt?= <117752024+daniel-rdt@users.noreply.github.com> Date: Thu, 26 Jun 2025 17:22:16 +0200 Subject: [PATCH 25/49] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Gilon Co-authored-by: Jonas Hörsch --- doc/release_notes.rst | 2 +- rules/retrieve.smk | 2 +- rules/solve_myopic.smk | 4 ++-- scripts/add_electricity.py | 18 +++--------------- scripts/build_renewable_profiles_pecd.py | 13 ++++++------- scripts/clean_pecd_data.py | 5 ++--- scripts/retrieve_tyndp_pecd_data.py | 2 +- 7 files changed, 16 insertions(+), 30 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 7cb970c9df..be0d654ab4 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,7 +13,7 @@ Release Notes **Changes** -* Add processing and preparation of TYNDP 2024 PECD renewable profiles instead of default renewable profiles using atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. +* Add processing and preparation of TYNDP 2024 PECD renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards. diff --git a/rules/retrieve.smk b/rules/retrieve.smk index e23c6fa4bf..fcacc4428c 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -187,8 +187,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" rule retrieve_tyndp_pecd_data: params: + # TODO Integrate into Zenodo tyndp data bundle tyndp_bundle="data/tyndp_2024_bundle", - # TODO Integrate into Zenodo tyndp data bundle output: dir=directory("data/tyndp_2024_bundle/PECD"), log: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index fb37bc164b..070a7e8cc2 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -69,7 +69,7 @@ def input_profile_tech_brownfield(w): } -def input_profile_tech_brownfied_pecd(w): +def input_profile_tech_brownfield_pecd(w): return { f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") for tech in pecd_renewable_profiles(w) @@ -94,7 +94,7 @@ rule add_brownfield: ), input: unpack(input_profile_tech_brownfield), - unpack(input_profile_tech_brownfied_pecd), + unpack(input_profile_tech_brownfield_pecd), simplify_busmap=resources("busmap_base_s.csv"), cluster_busmap=resources("busmap_base_s_{clusters}.csv"), network=resources( diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 1541df6e20..32b5649f3e 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1213,27 +1213,15 @@ def attach_stores( ) tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] + list(chain(*params.electricity["pecd_renewable_profiles"]["technologies"].values())) if params.electricity["pecd_renewable_profiles"]["enable"] else [] ) if len(tyndp_renewable_carriers) > 0: logger.info( - f"Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'. They will be attached later on with TYNDP data." + f"Skipping renewable carriers - they will be attached later with TYNDP data: {', '.join(tyndp_renewable_carriers)}" ) - renewable_carriers = set( - [ - carrier - for carrier in params.electricity["renewable_carriers"] - if carrier not in tyndp_renewable_carriers - ] - ) + renewable_carriers = set(params.electricity["renewable_carriers"]).difference(tyndp_renewable_carriers) extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 39197fb9c5..a7d102e1a2 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: MIT """ -Creates renewable profiles for each region from PECD and TYNDP input data containing the available -generation time series (based on PECD weather data) from the node for onshore wind, AC-connected offshore wind, -DC-connected offshore wind and solar PV generators. +Create renewable profiles for each region from PECD and TYNDP datasets, building on PECD climate data. The available +generation time series are read for each node across all renewable technologies, including onshore wind, AC-connected offshore wind, +DC-connected offshore wind, and solar PV generators. .. note:: Hydroelectric profiles will be built in script :mod:`build_hydro_profiles_PECD`. Not yet implemented. @@ -40,7 +40,7 @@ snakemake = mock_snakemake( "build_renewable_profiles_pecd", clusters="all", - technology="offwind-ac", + technology="offwind", ) configure_logging(snakemake) set_scenario_config(snakemake) @@ -58,12 +58,11 @@ if int(year) not in [2030, 2040, 2050]: year = np.clip(10 * (year // 10), 2030, 2050) logger.warning( - "Planning horizon doesn't match available TYNDP PECD data. " - f"Falling back to previous available year {year}." + f"TYNDP PECD data unavailable for planning horizon. Falling back to previous available year {year}." ) if year == 2050: logger.warning( - "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." + "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data." ) year = 2040 diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 98f6826dec..ba30ecb3e1 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -37,7 +37,7 @@ def read_pecd_file( - node: list, + node: str, fn_pecd: str, cyear: str, pyear: str, @@ -140,8 +140,7 @@ def read_pecd_file( pecd_df = ( pd.concat(demand, axis=1) - .reindex(nodes, axis=1) - .fillna(0.0) # include missing node data with empty columns + .reindex(nodes, axis=1, fill_value=0.0) # include missing node data with empty columns .rename( columns=lambda x: x.replace("UK", "GB") ) # replace UK with GB for naming convention diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index a24bbc3675..67a5a84168 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -4,7 +4,7 @@ """ The TYNDP PECD data contains input data for the 2024 TYNDP scenario building process. -This rule downloads the TYNDP PECD data from `a gdrive and extracts it in the ``data/tyndp_2024_bundle`` +This rule downloads the TYNDP PECD data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` subdirectory, such that all files of the TYNDP bundle are stored in it. **Outputs** From faf71b5fbdf32a114b49da5334c1b073c556ce03 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 15:22:30 +0000 Subject: [PATCH 26/49] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 10 ++++++++-- scripts/clean_pecd_data.py | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 32b5649f3e..a53cfaeae8 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1213,7 +1213,11 @@ def attach_stores( ) tyndp_renewable_carriers = ( - list(chain(*params.electricity["pecd_renewable_profiles"]["technologies"].values())) + list( + chain( + *params.electricity["pecd_renewable_profiles"]["technologies"].values() + ) + ) if params.electricity["pecd_renewable_profiles"]["enable"] else [] ) @@ -1221,7 +1225,9 @@ def attach_stores( logger.info( f"Skipping renewable carriers - they will be attached later with TYNDP data: {', '.join(tyndp_renewable_carriers)}" ) - renewable_carriers = set(params.electricity["renewable_carriers"]).difference(tyndp_renewable_carriers) + renewable_carriers = set(params.electricity["renewable_carriers"]).difference( + tyndp_renewable_carriers + ) extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index ba30ecb3e1..9c2f19b599 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -140,7 +140,9 @@ def read_pecd_file( pecd_df = ( pd.concat(demand, axis=1) - .reindex(nodes, axis=1, fill_value=0.0) # include missing node data with empty columns + .reindex( + nodes, axis=1, fill_value=0.0 + ) # include missing node data with empty columns .rename( columns=lambda x: x.replace("UK", "GB") ) # replace UK with GB for naming convention From 81361210bbcc703f2dcc1f1a560e8a7713fa00c0 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 26 Jun 2025 17:41:19 +0200 Subject: [PATCH 27/49] fix: import chain from itertools --- scripts/add_electricity.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a53cfaeae8..a20c0c9677 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -50,6 +50,7 @@ import logging from collections.abc import Iterable +from itertools import chain from typing import Any import geopandas as gpd From a67fb078b38acf72e99a4eb946bf0a1c0c8b172c Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 26 Jun 2025 18:06:52 +0200 Subject: [PATCH 28/49] refactor: generalize get tyndp renewable carriers --- rules/solve_myopic.smk | 8 ++++++-- rules/solve_perfect.smk | 4 +++- scripts/add_brownfield.py | 14 +++---------- scripts/add_electricity.py | 34 ++++++++++++++++++++++++-------- scripts/add_existing_baseyear.py | 18 +++++++---------- 5 files changed, 45 insertions(+), 33 deletions(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 070a7e8cc2..e880b962ac 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -7,7 +7,9 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), - electricity=config_provider("electricity"), + pecd_renewable_profiles=config_provider( + "electricity", "pecd_renewable_profiles" + ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), @@ -84,7 +86,9 @@ rule add_brownfield: ), threshold_capacity=config_provider("existing_capacities", "threshold_capacity"), snapshots=config_provider("snapshots"), - electricity=config_provider("electricity"), + pecd_renewable_profiles=config_provider( + "electricity", "pecd_renewable_profiles" + ), drop_leap_day=config_provider("enable", "drop_leap_day"), carriers=config_provider("electricity", "renewable_carriers"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index ce9f905412..0b74cb35d1 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -5,7 +5,9 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), - electricity=config_provider("electricity"), + pecd_renewable_profiles=config_provider( + "electricity", "pecd_renewable_profiles" + ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 127c24d0bd..06e129ed56 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -19,7 +19,7 @@ set_scenario_config, update_config_from_wildcards, ) -from scripts.add_electricity import flatten, sanitize_carriers +from scripts.add_electricity import flatten, get_tyndp_res_carriers, sanitize_carriers from scripts.add_existing_baseyear import add_build_year_to_new_assets logger = logging.getLogger(__name__) @@ -356,16 +356,8 @@ def update_dynamic_ptes_capacity( n = pypsa.Network(snakemake.input.network) - tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] - if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] - else [] + tyndp_renewable_carriers = get_tyndp_res_carriers( + snakemake.params.pecd_renewable_profiles ) adjust_renewable_profiles( diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a20c0c9677..38b5f9d3e3 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -190,6 +190,30 @@ def sanitize_carriers(n, config): n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors) +def get_tyndp_res_carriers(pecd_renewable_profiles: dict): + """ + Function to return all TYNDP renewable carriers specified in the configuration file for PECD profiles. + + The function makes sure TYNDP renewable carriers are only returned if PECD profiles are enabled. + + Parameters + ---------- + pecd_renewable_profiles : dict + Dictionary that contains all TYNDP renewable carrier for of the PECD profiles. + + Returns + ------- + tyndp_renewable_carriers : list + List of TYNDP renewable carriers. + """ + tyndp_renewable_carriers = ( + list(chain(*pecd_renewable_profiles["technologies"].values())) + if pecd_renewable_profiles["enable"] + else [] + ) + return tyndp_renewable_carriers + + def sanitize_locations(n): if "location" in n.buses.columns: n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x)) @@ -1213,14 +1237,8 @@ def attach_stores( params.link_length_factor, ) - tyndp_renewable_carriers = ( - list( - chain( - *params.electricity["pecd_renewable_profiles"]["technologies"].values() - ) - ) - if params.electricity["pecd_renewable_profiles"]["enable"] - else [] + tyndp_renewable_carriers = get_tyndp_res_carriers( + params.electricity["pecd_renewable_profiles"] ) if len(tyndp_renewable_carriers) > 0: logger.info( diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index c78d72bba8..3015db9635 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -23,7 +23,11 @@ set_scenario_config, update_config_from_wildcards, ) -from scripts.add_electricity import load_costs, sanitize_carriers +from scripts.add_electricity import ( + get_tyndp_res_carriers, + load_costs, + sanitize_carriers, +) from scripts.build_energy_totals import cartesian from scripts.definitions.heat_system import HeatSystem from scripts.prepare_sector_network import cluster_heat_buses, define_spatial @@ -749,16 +753,8 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector renewable_carriers = snakemake.params.carriers - tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] - if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] - else [] + tyndp_renewable_carriers = get_tyndp_res_carriers( + snakemake.params.pecd_renewable_profiles ) baseyear = snakemake.params.baseyear From b1931225b99223ee23d04fdc7ce25b77f8507743 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 13:35:55 +0200 Subject: [PATCH 29/49] fix: remove placeholder for renewable technology max capacities in build_renewable_profiles_pecd --- scripts/build_renewable_profiles_pecd.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index a7d102e1a2..2db9c11c2e 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -81,17 +81,4 @@ profiles.append(profile) ds = xr.merge(profiles) - - # TODO: Later on the max capacities for renewable technologies and regions can be added here - # profiles = xr.merge(profiles) - # p_nom_max = place_holder - # average_distance = place_holder - # ds = xr.merge( - # [ - # profiles, - # p_nom_max.rename("p_nom_max"), - # average_distance.rename("average_distance"), - # ] - # ) - ds.to_netcdf(snakemake.output.profile) From fed98dd08c6e07fb0586e6cf4bcd1b1579799f8f Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 13:47:36 +0200 Subject: [PATCH 30/49] doc: add information about PECD Version 3.1 to documentation --- doc/release_notes.rst | 2 +- scripts/retrieve_tyndp_pecd_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 66ff963092..2882f103f0 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,7 +13,7 @@ Release Notes **Changes** -* Add processing and preparation of TYNDP 2024 PECD renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. +* Add processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards. diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 67a5a84168..63d05d5d62 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -4,7 +4,7 @@ """ The TYNDP PECD data contains input data for the 2024 TYNDP scenario building process. -This rule downloads the TYNDP PECD data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` +This rule downloads the TYNDP PECD v3.1 data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` subdirectory, such that all files of the TYNDP bundle are stored in it. **Outputs** From 0a36daa061195315022e96b2f31e76bda2219605 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 14:00:30 +0200 Subject: [PATCH 31/49] fix: remove unnecessary params from pecd rules --- rules/build_electricity.smk | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 963535f5ed..8aa5b8b2ea 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -395,7 +395,6 @@ rule build_renewable_profiles: rule clean_pecd_data: params: - scenario=config_provider("tyndp_scenario"), snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), input: @@ -430,9 +429,6 @@ def input_data_pecd(w): rule build_renewable_profiles_pecd: params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - renewable=config_provider("renewable"), planning_horizons=config_provider("scenario", "planning_horizons"), input: unpack(input_data_pecd), From b98ecccff4bf16690bedc7df581a094bcde73d20 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 14:05:17 +0200 Subject: [PATCH 32/49] feat: change fn_pecd to dir_pecd and add comment on skiprows --- rules/build_electricity.smk | 2 +- scripts/clean_pecd_data.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 8aa5b8b2ea..4e1dcab164 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -400,7 +400,7 @@ rule clean_pecd_data: input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), - fn_pecd="data/tyndp_2024_bundle/PECD", + dir_pecd="data/tyndp_2024_bundle/PECD", output: pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), log: diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 9c2f19b599..0bfbaaae71 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -38,20 +38,20 @@ def read_pecd_file( node: str, - fn_pecd: str, + dir_pecd: str, cyear: str, pyear: str, technology: str, sns: pd.DatetimeIndex, ): - fn = Path(fn_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + fn = Path(dir_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") if not os.path.isfile(fn): return None pecd_bus = pd.read_csv( fn, - skiprows=10, + skiprows=10, # first ten rows contain only file metadata usecols=lambda name: name == "Date" or name == "Hour" or name == str(cyear) @@ -116,7 +116,7 @@ def read_pecd_file( nodes = ( offshore_buses.index if pecd_tech == "Wind_Offshore" else onshore_buses.index ) - fn_pecd = snakemake.input.fn_pecd + dir_pecd = snakemake.input.dir_pecd # Load and prep electricity demand tqdm_kwargs = { @@ -128,7 +128,7 @@ def read_pecd_file( func = partial( read_pecd_file, - fn_pecd=fn_pecd, + dir_pecd=dir_pecd, cyear=cyear, pyear=pyear, technology=pecd_tech, From 580eaaa025e64c45916215ca19adf9952af3d77a Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 14:16:28 +0200 Subject: [PATCH 33/49] refactor: change to if not continue style to reduce diff --- scripts/add_existing_baseyear.py | 81 ++++++++++++++++---------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 781708f1e7..0022b3e8af 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -113,46 +113,47 @@ def add_existing_renewables( irena = irena.unstack().reset_index() for carrier, tech in tech_map.items(): - if carrier in renewable_carriers: - df = ( - irena[irena.Technology.str.contains(tech)] - .drop(columns=["Technology"]) - .set_index("Country") - ) - df.columns = df.columns.astype(int) - - # calculate yearly differences - df.insert(loc=0, value=0.0, column="1999") - df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0) - - # distribute capacities among generators potential (p_nom_max) - gen_i = n.generators.query("carrier == @carrier").index - carrier_gens = n.generators.loc[gen_i] - res_capacities = [] - for country, group in carrier_gens.groupby( - carrier_gens.bus.map(n.buses.country) - ): - fraction = group.p_nom_max / group.p_nom_max.sum() - res_capacities.append(cartesian(df.loc[country], fraction)) - res_capacities = pd.concat(res_capacities, axis=1).T - - for year in res_capacities.columns: - for gen in res_capacities.index: - bus_bin = re.sub(f" {carrier}.*", "", gen) - bus, bin_id = bus_bin.rsplit(" ", maxsplit=1) - name = f"{bus_bin} {carrier}-{year}" - capacity = res_capacities.loc[gen, year] - if capacity > 0.0: - cost_key = carrier.split("-", maxsplit=1)[0] - df_agg.at[name, "Fueltype"] = carrier - df_agg.at[name, "Capacity"] = capacity - df_agg.at[name, "DateIn"] = year - df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"] - df_agg.at[name, "DateOut"] = ( - year + costs.at[cost_key, "lifetime"] - 1 - ) - df_agg.at[name, "bus"] = bus - df_agg.at[name, "resource_class"] = bin_id + if carrier not in renewable_carriers: + continue + df = ( + irena[irena.Technology.str.contains(tech)] + .drop(columns=["Technology"]) + .set_index("Country") + ) + df.columns = df.columns.astype(int) + + # calculate yearly differences + df.insert(loc=0, value=0.0, column="1999") + df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0) + + # distribute capacities among generators potential (p_nom_max) + gen_i = n.generators.query("carrier == @carrier").index + carrier_gens = n.generators.loc[gen_i] + res_capacities = [] + for country, group in carrier_gens.groupby( + carrier_gens.bus.map(n.buses.country) + ): + fraction = group.p_nom_max / group.p_nom_max.sum() + res_capacities.append(cartesian(df.loc[country], fraction)) + res_capacities = pd.concat(res_capacities, axis=1).T + + for year in res_capacities.columns: + for gen in res_capacities.index: + bus_bin = re.sub(f" {carrier}.*", "", gen) + bus, bin_id = bus_bin.rsplit(" ", maxsplit=1) + name = f"{bus_bin} {carrier}-{year}" + capacity = res_capacities.loc[gen, year] + if capacity > 0.0: + cost_key = carrier.split("-", maxsplit=1)[0] + df_agg.at[name, "Fueltype"] = carrier + df_agg.at[name, "Capacity"] = capacity + df_agg.at[name, "DateIn"] = year + df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"] + df_agg.at[name, "DateOut"] = ( + year + costs.at[cost_key, "lifetime"] - 1 + ) + df_agg.at[name, "bus"] = bus + df_agg.at[name, "resource_class"] = bin_id df_agg["resource_class"] = df_agg["resource_class"].fillna(0) From a55cfbf1ef8bafb2ba651be68d6617e716984c93 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 14:44:31 +0200 Subject: [PATCH 34/49] feat: improve pecd profile configuration by making profile names explicit and improving documentation --- config/config.default.yaml | 2 +- config/config.tyndp.yaml | 2 +- config/test/config.tyndp.yaml | 2 +- doc/configtables/electricity.csv | 4 ++-- scripts/clean_pecd_data.py | 13 ++----------- 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 038de075d8..d8ddb03f3a 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -116,7 +116,7 @@ electricity: pecd_renewable_profiles: enable: false technologies: - offwind: + Wind_Offshore: - offwind-ac-fb-r - offwind-ac-fl-r - offwind-dc-fb-r diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index af2822fd44..1d94f1349d 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -43,7 +43,7 @@ electricity: pecd_renewable_profiles: enable: true technologies: - offwind: + Wind_Offshore: - offwind-ac-fb-r - offwind-ac-fl-r - offwind-dc-fb-r diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index eab5f431c2..beed2b657d 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -41,7 +41,7 @@ electricity: pecd_renewable_profiles: enable: true technologies: - offwind: + Wind_Offshore: - offwind-ac-fb-r - offwind-ac-fl-r - offwind-dc-fb-r diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 3af8782456..bbd5199c05 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -31,9 +31,9 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite ,,, renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. pecd_renewable_profiles,,, --- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_profile_name`` can be any of {Wind_Offshore, Wind_Onshore, LFSolarPV}. -- technologies,,, --- -- {pecd_profile},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. Carriers not listed use default renewable profiles. +-- -- {pecd_profile_name},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. These TYNDP renewable carriers should also be listed in ``renewable_carriers``. Carriers not listed use default renewable profiles. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 0bfbaaae71..ebe8da9c38 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -97,18 +97,9 @@ def read_pecd_file( # Planning year pyear = str(snakemake.wildcards.planning_horizons) - # Technology as in PECD terminology # TODO: find solution for solar profiles being differentiated between Utility and Rooftop for Italy - pecd_tech_dict = { - "offwind-ac": "Wind_Offshore", - "offwind-dc": "Wind_Offshore", - "offwind-float": "Wind_Offshore", - "offwind": "Wind_Offshore", - "onwind": "Wind_Onshore", - "solar": "LFSolarPV", - "solar-hsat": "LFSolarPV", - } - pecd_tech = pecd_tech_dict[snakemake.wildcards.technology] + # Technology as in PECD terminology + pecd_tech = snakemake.wildcards.technology offshore_buses = pd.read_excel(snakemake.input.offshore_buses, index_col=0) onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) From decd1868b02efd90b576793a36f997558a3216ad Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 15:19:45 +0200 Subject: [PATCH 35/49] refactor: simplify tyndp_renewable_carriers also in smk files --- Snakefile | 1 + rules/build_electricity.smk | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Snakefile b/Snakefile index f21e0c5568..a6c449205a 100644 --- a/Snakefile +++ b/Snakefile @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: MIT +from itertools import chain from pathlib import Path import yaml from os.path import normpath, exists, join diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 4e1dcab164..455a7762af 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -788,13 +788,13 @@ rule cluster_network: def tyndp_renewable_carriers(w): return ( - [ - subcarrier - for carrier in config_provider( - "electricity", "pecd_renewable_profiles", "technologies" - )(w).values() - for subcarrier in carrier - ] + list( + chain( + *config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w).values() + ) + ) if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) else [] ) From ad4236461bccd8462a1d33a5715b97abe666c5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20R=C3=BCdt?= <117752024+daniel-rdt@users.noreply.github.com> Date: Tue, 1 Jul 2025 11:55:57 +0200 Subject: [PATCH 36/49] Apply suggestions from code review Co-authored-by: Thomas Gilon --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 38b5f9d3e3..df4be84a12 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -199,7 +199,7 @@ def get_tyndp_res_carriers(pecd_renewable_profiles: dict): Parameters ---------- pecd_renewable_profiles : dict - Dictionary that contains all TYNDP renewable carrier for of the PECD profiles. + Dictionary that contains all TYNDP renewable carriers of the PECD profiles. Returns ------- From 3a10294c116d4f3063db687de3bb65778872ab74 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 1 Jul 2025 13:43:08 +0200 Subject: [PATCH 37/49] feat: adjust mock snakemake for changed first level config of pecd profiles --- scripts/build_renewable_profiles_pecd.py | 2 +- scripts/clean_pecd_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 2db9c11c2e..67646349b6 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -40,7 +40,7 @@ snakemake = mock_snakemake( "build_renewable_profiles_pecd", clusters="all", - technology="offwind", + technology="Wind_Offshore", ) configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index ebe8da9c38..c582b08b1d 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -78,7 +78,7 @@ def read_pecd_file( snakemake = mock_snakemake( "clean_pecd_data", clusters="all", - technology="offwind-ac", + technology="Wind_Offshore", planning_horizons=2030, ) configure_logging(snakemake) From a4d8cf1b18af4a4e2dc93d56de61f7dff58ae179 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 1 Jul 2025 13:49:19 +0200 Subject: [PATCH 38/49] feat: harmonize order of config options to default config --- config/config.tyndp.yaml | 6 ++++-- config/test/config.tyndp.yaml | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 1d94f1349d..da9cfbf824 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -39,7 +39,9 @@ co2_budget: electricity: base_network: tyndp-raw - transmission_limit: v1.0 + + renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro] + pecd_renewable_profiles: enable: true technologies: @@ -53,7 +55,7 @@ electricity: - offwind-h2-fb-oh - offwind-h2-fl-oh - renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro] + transmission_limit: v1.0 links: p_max_pu: 1.0 diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index beed2b657d..1afe25f2a7 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -37,7 +37,15 @@ co2_budget: electricity: base_network: tyndp-raw - transmission_limit: v1.0 + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [battery] + Store: [H2] + Link: [H2 pipeline] + + renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + pecd_renewable_profiles: enable: true technologies: @@ -51,13 +59,7 @@ electricity: - offwind-h2-fb-oh - offwind-h2-fl-oh - extendable_carriers: - Generator: [OCGT] - StorageUnit: [battery] - Store: [H2] - Link: [H2 pipeline] - - renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + transmission_limit: v1.0 atlite: default_cutout: europe-2013-03-sarah3-era5 From b604850e5fc9126e2b5316ccd17a072c580221df Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 1 Jul 2025 13:53:20 +0200 Subject: [PATCH 39/49] refactor: move get_tyndp_res_carriers to _helpers.py --- scripts/_helpers.py | 25 +++++++++++++++++++++++++ scripts/add_brownfield.py | 3 ++- scripts/add_electricity.py | 26 +------------------------- scripts/add_existing_baseyear.py | 2 +- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 8c44416b58..1eb1c17e3e 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -10,6 +10,7 @@ import re import time from functools import partial, wraps +from itertools import chain from pathlib import Path from tempfile import NamedTemporaryFile from typing import Callable, Union @@ -1125,3 +1126,27 @@ def extract_grid_data_tyndp( h2_grid.index = h2_grid.apply(make_index, axis=1, args=(carrier,)) return h2_grid + + +def get_tyndp_res_carriers(pecd_renewable_profiles: dict): + """ + Function to return all TYNDP renewable carriers specified in the configuration file for PECD profiles. + + The function makes sure TYNDP renewable carriers are only returned if PECD profiles are enabled. + + Parameters + ---------- + pecd_renewable_profiles : dict + Dictionary that contains all TYNDP renewable carriers of the PECD profiles. + + Returns + ------- + tyndp_renewable_carriers : list + List of TYNDP renewable carriers. + """ + tyndp_renewable_carriers = ( + list(chain(*pecd_renewable_profiles["technologies"].values())) + if pecd_renewable_profiles["enable"] + else [] + ) + return tyndp_renewable_carriers diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 06e129ed56..17e9ee75ae 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -15,11 +15,12 @@ from scripts._helpers import ( configure_logging, get_snapshots, + get_tyndp_res_carriers, sanitize_custom_columns, set_scenario_config, update_config_from_wildcards, ) -from scripts.add_electricity import flatten, get_tyndp_res_carriers, sanitize_carriers +from scripts.add_electricity import flatten, sanitize_carriers from scripts.add_existing_baseyear import add_build_year_to_new_assets logger = logging.getLogger(__name__) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index df4be84a12..d4dc0943db 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -50,7 +50,6 @@ import logging from collections.abc import Iterable -from itertools import chain from typing import Any import geopandas as gpd @@ -64,6 +63,7 @@ from scripts._helpers import ( configure_logging, get_snapshots, + get_tyndp_res_carriers, rename_techs, set_scenario_config, update_p_nom_max, @@ -190,30 +190,6 @@ def sanitize_carriers(n, config): n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors) -def get_tyndp_res_carriers(pecd_renewable_profiles: dict): - """ - Function to return all TYNDP renewable carriers specified in the configuration file for PECD profiles. - - The function makes sure TYNDP renewable carriers are only returned if PECD profiles are enabled. - - Parameters - ---------- - pecd_renewable_profiles : dict - Dictionary that contains all TYNDP renewable carriers of the PECD profiles. - - Returns - ------- - tyndp_renewable_carriers : list - List of TYNDP renewable carriers. - """ - tyndp_renewable_carriers = ( - list(chain(*pecd_renewable_profiles["technologies"].values())) - if pecd_renewable_profiles["enable"] - else [] - ) - return tyndp_renewable_carriers - - def sanitize_locations(n): if "location" in n.buses.columns: n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x)) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 0022b3e8af..5ffd2f0f81 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -19,12 +19,12 @@ from scripts._helpers import ( configure_logging, + get_tyndp_res_carriers, sanitize_custom_columns, set_scenario_config, update_config_from_wildcards, ) from scripts.add_electricity import ( - get_tyndp_res_carriers, load_costs, sanitize_carriers, ) From 8bd32c814df4d0f31dcef6092357149d74c48d60 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 1 Jul 2025 13:56:06 +0200 Subject: [PATCH 40/49] doc: add CSP to list of PECD profile names --- doc/configtables/electricity.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index bbd5199c05..a9d6442876 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -31,7 +31,7 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite ,,, renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. pecd_renewable_profiles,,, --- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_profile_name`` can be any of {Wind_Offshore, Wind_Onshore, LFSolarPV}. +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_profile_name`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. -- technologies,,, -- -- {pecd_profile_name},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. These TYNDP renewable carriers should also be listed in ``renewable_carriers``. Carriers not listed use default renewable profiles. estimate_renewable_capacities,,, From 0c63140c19b989c44a2b4ac32707632817749cda Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 2 Jul 2025 09:27:28 +0200 Subject: [PATCH 41/49] doc: add todo for pecd data retrieval --- scripts/retrieve_tyndp_pecd_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 63d05d5d62..8dc9ffdb1b 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -23,6 +23,7 @@ logger = logging.getLogger(__name__) # Define the base URL +# TODO: retrieve_tyndp_pecd_data needs to be deprecated once PECD data is added to the TYNDP data bundle url = "https://storage.googleapis.com/open-tyndp-data-store/PECD.zip" if __name__ == "__main__": From 3e4b3f87ba841d8b759ac5dfbf5ef5d92938fd31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20R=C3=BCdt?= <117752024+daniel-rdt@users.noreply.github.com> Date: Wed, 2 Jul 2025 16:14:04 +0200 Subject: [PATCH 42/49] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonas Hörsch --- rules/build_sector.smk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index b01e07dafc..1d2e1b8224 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1234,13 +1234,13 @@ def input_profile_offwind(w): def pecd_renewable_profiles(w): return ( - [ - carrier - for carrier in config_provider( - "electricity", "pecd_renewable_profiles", "technologies" - )(w).keys() - ] - if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) + list( + config_provider( + "electricity", + "pecd_renewable_profiles", + "technologies" + )(w) + ) if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) else [] ) From 76c8cc8301c96bf461e8048f96aeb2ae5a4583e5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 14:14:29 +0000 Subject: [PATCH 43/49] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_sector.smk | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1d2e1b8224..ec23faaa0a 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1234,13 +1234,10 @@ def input_profile_offwind(w): def pecd_renewable_profiles(w): return ( - list( - config_provider( - "electricity", - "pecd_renewable_profiles", - "technologies" - )(w) - ) if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) + list( + config_provider("electricity", "pecd_renewable_profiles", "technologies")(w) + ) + if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) else [] ) From 86d955ecba831db86e541cf6dea83e31d79b7781 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 2 Jul 2025 16:25:27 +0200 Subject: [PATCH 44/49] doc: improve documentation of pecd_renewable_profiles option --- doc/configtables/electricity.csv | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index a9d6442876..95485753e0 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -31,9 +31,10 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite ,,, renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. pecd_renewable_profiles,,, --- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_profile_name`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. -- technologies,,, --- -- {pecd_profile_name},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. These TYNDP renewable carriers should also be listed in ``renewable_carriers``. Carriers not listed use default renewable profiles. +-- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tydnp_renewable_carriers``. +-- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. Carriers not listed use the default renewable profiles. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. From f068db4cc468d00cd9f7b4ff54aed5e2af3ccd51 Mon Sep 17 00:00:00 2001 From: Jonas Hoersch Date: Thu, 3 Jul 2025 11:40:03 +0200 Subject: [PATCH 45/49] Add missing input, use branch --- rules/build_sector.smk | 13 ++++--------- rules/retrieve.smk | 1 + 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index ec23faaa0a..a8d82a1a79 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1232,15 +1232,10 @@ def input_profile_offwind(w): } -def pecd_renewable_profiles(w): - return ( - list( - config_provider("electricity", "pecd_renewable_profiles", "technologies")(w) - ) - if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) - else [] - ) - +pecd_renewable_profiles = branch( + config_provider("electricity", "pecd_renewable_profiles", "enable"), + config_provider("electricity", "pecd_renewable_profiles", "technologies") +) def input_profile_pecd(w): return { diff --git a/rules/retrieve.smk b/rules/retrieve.smk index fcacc4428c..514551171c 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -179,6 +179,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" h2_reference_grid="data/tyndp_2024_bundle/Line data/ReferenceGrid_Hydrogen.xlsx", electricity_demand=directory("data/tyndp_2024_bundle/Demand Profiles"), h2_imports="data/tyndp_2024_bundle/Hydrogen/H2 IMPORTS GENERATORS PROPERTIES.xlsx", + offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", log: "logs/retrieve_tyndp_bundle.log", retries: 2 From d6f3ae1b4ec876d560d7c86f9ee98603fed34d24 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 3 Jul 2025 09:40:28 +0000 Subject: [PATCH 46/49] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_sector.smk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index a8d82a1a79..1a9605850e 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1234,9 +1234,10 @@ def input_profile_offwind(w): pecd_renewable_profiles = branch( config_provider("electricity", "pecd_renewable_profiles", "enable"), - config_provider("electricity", "pecd_renewable_profiles", "technologies") + config_provider("electricity", "pecd_renewable_profiles", "technologies"), ) + def input_profile_pecd(w): return { f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") From 48ab5d5bc5af9ee237beaeb32bfb3ce7e0de352e Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 3 Jul 2025 12:49:10 +0200 Subject: [PATCH 47/49] feat: make estimate renewable capacities explicit from config for selected technologies and remove techs covered by TYNDP carriers from list --- config/config.default.yaml | 4 ++ config/config.tyndp.yaml | 6 ++ config/test/config.tyndp.yaml | 6 ++ doc/configtables/electricity.csv | 101 ++++++++++++++++--------------- scripts/add_electricity.py | 2 +- 5 files changed, 68 insertions(+), 51 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index d8ddb03f3a..7a9db61877 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -131,6 +131,10 @@ electricity: from_gem: true year: 2020 expansion_limit: false + technologies: + - Offshore + - Onshore + - PV technology_mapping: Offshore: offwind-ac Onshore: onwind diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index da9cfbf824..8eb93c8985 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -55,6 +55,12 @@ electricity: - offwind-h2-fb-oh - offwind-h2-fl-oh + estimate_renewable_capacities: + # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation + technologies: + - Onshore + - PV + transmission_limit: v1.0 links: diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 1afe25f2a7..5b2acbdd19 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -59,6 +59,12 @@ electricity: - offwind-h2-fb-oh - offwind-h2-fl-oh + estimate_renewable_capacities: + # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation + technologies: + - Onshore + - PV + transmission_limit: v1.0 atlite: diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 95485753e0..72f8712196 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -1,50 +1,51 @@ -,Unit,Values,Description -voltages,kV,"Any subset of {220., 300., 330., 380., 400., 500., 750.}",Voltage levels to consider -base_network, --, "Any value in {'entsoegridkit', 'osm-prebuilt', 'osm-raw}", "Specify the underlying base network, i.e. GridKit (based on ENTSO-E web map extract, OpenStreetMap (OSM) prebuilt or raw (built from raw OSM data), takes longer." -osm-prebuilt-version, --, "float, any value in range 0.1-0.6", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release." -gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``. -gaslimit,MWhth,float or false,Global gas usage limit -co2limit_enable,bool,true or false,"Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit`` in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks." -co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions -co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard. -operational_reserve,,,Settings for reserve requirements following `GenX `_ -,,, --- activate,bool,true or false,Whether to take operational reserve requirements into account during optimisation --- epsilon_load,--,float,share of total load --- epsilon_vres,--,float,share of total renewable supply --- contingency,MW,float,fixed reserve capacity -max_hours,,, --- battery,h,float,Maximum state of charge capacity of the battery in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. --- H2,h,float,Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. -extendable_carriers,,, --- Generator,--,Any extendable carrier,"Defines existing or non-existing conventional and renewable power plants to be extendable during the optimization. Conventional generators can only be built/expanded where already existent today. If a listed conventional carrier is not included in the ``conventional_carriers`` list, the lower limit of the capacity expansion is set to 0." --- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. --- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. --- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``. -powerplants_filter,--,"use `pandas.query `_ strings here, e.g. ``Country not in ['Germany']``",Filter query for the default powerplant database. -,,, -custom_powerplants,--,"use `pandas.query `_ strings here, e.g. ``Country in ['Germany']``",Filter query for the custom powerplant database. -,,, -everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to add to every node in the model with zero initial capacity. To be used in combination with ``extendable_carriers`` to allow for building conventional powerplants irrespective of existing locations." -,,, -conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound." -,,, -renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. -pecd_renewable_profiles,,, --- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. --- technologies,,, --- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tydnp_renewable_carriers``. --- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. Carriers not listed use the default renewable profiles. -estimate_renewable_capacities,,, --- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. --- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. --- year,--,bool,Renewable capacities are based on existing capacities reported by IRENA (IRENASTAT) for the specified year --- expansion_limit,--,float or false,"Artificially limit maximum IRENA capacities to a factor. For example, an ``expansion_limit: 1.1`` means 110% of capacities . If false are chosen, the estimated renewable potentials determine by the workflow are used." --- technology_mapping,,,Mapping between PyPSA-Eur and powerplantmatching technology names --- -- Offshore,--,"{onwind}","PyPSA-Eur carrier that is considered for existing onshore wind capacities (IRENA, GEM)." --- -- Offshore,--,"Any of {offwind-ac, offwind-dc, offwind-float}","PyPSA-Eur carrier that is considered for existing offshore wind technology (IRENA, GEM)." --- -- PV,--,{solar},"PyPSA-Eur carrier that is considered for existing solar PV capacities (IRENA, GEM)." -autarky,,, --- enable,bool,true or false,Require each node to be autarkic by removing all lines and links. --- by_country,bool,true or false,Require each country to be autarkic by removing all cross-border lines and links. ``electricity: autarky`` must be enabled. -transmission_limit,str,"Values like 'vopt', 'v1.25', 'copt', 'c1.25'","Limit on transmission expansion. The first part can be ``v`` (for setting a limit on line volume) or ``c`` (for setting a limit on line cost). The second part can be ``opt`` or a float bigger than one (e.g. 1.25). If ``opt`` is chosen line expansion is optimised according to its capital cost (where the choice ``v`` only considers overhead costs for HVDC transmission lines, while ``c`` uses more accurate costs distinguishing between overhead and underwater sections and including inverter pairs). The setting ``v1.25`` will limit the total volume of line expansion to 25% of currently installed capacities weighted by individual line lengths. The setting ``c1.25`` will allow to build a transmission network that costs no more than 25 % more than the current system." +,Unit,Values,Description +voltages,kV,"Any subset of {220., 300., 330., 380., 400., 500., 750.}",Voltage levels to consider +base_network, --, "Any value in {'entsoegridkit', 'osm-prebuilt', 'osm-raw}", "Specify the underlying base network, i.e. GridKit (based on ENTSO-E web map extract, OpenStreetMap (OSM) prebuilt or raw (built from raw OSM data), takes longer." +osm-prebuilt-version, --, "float, any value in range 0.1-0.6", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release." +gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``. +gaslimit,MWhth,float or false,Global gas usage limit +co2limit_enable,bool,true or false,"Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit`` in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks." +co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions +co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard. +operational_reserve,,,Settings for reserve requirements following `GenX `_ +,,, +-- activate,bool,true or false,Whether to take operational reserve requirements into account during optimisation +-- epsilon_load,--,float,share of total load +-- epsilon_vres,--,float,share of total renewable supply +-- contingency,MW,float,fixed reserve capacity +max_hours,,, +-- battery,h,float,Maximum state of charge capacity of the battery in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. +-- H2,h,float,Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. +extendable_carriers,,, +-- Generator,--,Any extendable carrier,"Defines existing or non-existing conventional and renewable power plants to be extendable during the optimization. Conventional generators can only be built/expanded where already existent today. If a listed conventional carrier is not included in the ``conventional_carriers`` list, the lower limit of the capacity expansion is set to 0." +-- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. +-- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. +-- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``. +powerplants_filter,--,"use `pandas.query `_ strings here, e.g. ``Country not in ['Germany']``",Filter query for the default powerplant database. +,,, +custom_powerplants,--,"use `pandas.query `_ strings here, e.g. ``Country in ['Germany']``",Filter query for the custom powerplant database. +,,, +everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to add to every node in the model with zero initial capacity. To be used in combination with ``extendable_carriers`` to allow for building conventional powerplants irrespective of existing locations." +,,, +conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound." +,,, +renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. +pecd_renewable_profiles,,, +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. +-- technologies,,, +-- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tydnp_renewable_carriers``. +-- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. Carriers not listed use the default renewable profiles. +estimate_renewable_capacities,,, +-- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. +-- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. +-- year,--,bool,Renewable capacities are based on existing capacities reported by IRENA (IRENASTAT) for the specified year +-- expansion_limit,--,float or false,"Artificially limit maximum IRENA capacities to a factor. For example, an ``expansion_limit: 1.1`` means 110% of capacities . If false are chosen, the estimated renewable potentials determine by the workflow are used." +-- technologies,--,list,Switch to select the technologies for which renewable capacities are estimated for overnight foresight. Technologies covered by specified TYNDP renewable carriers need to be removed. +-- technology_mapping,,,Mapping between PyPSA-Eur and powerplantmatching technology names +-- -- Offshore,--,"{onwind}","PyPSA-Eur carrier that is considered for existing onshore wind capacities (IRENA, GEM)." +-- -- Offshore,--,"Any of {offwind-ac, offwind-dc, offwind-float}","PyPSA-Eur carrier that is considered for existing offshore wind technology (IRENA, GEM)." +-- -- PV,--,{solar},"PyPSA-Eur carrier that is considered for existing solar PV capacities (IRENA, GEM)." +autarky,,, +-- enable,bool,true or false,Require each node to be autarkic by removing all lines and links. +-- by_country,bool,true or false,Require each country to be autarkic by removing all cross-border lines and links. ``electricity: autarky`` must be enabled. +transmission_limit,str,"Values like 'vopt', 'v1.25', 'copt', 'c1.25'","Limit on transmission expansion. The first part can be ``v`` (for setting a limit on line volume) or ``c`` (for setting a limit on line cost). The second part can be ``opt`` or a float bigger than one (e.g. 1.25). If ``opt`` is chosen line expansion is optimised according to its capital cost (where the choice ``v`` only considers overhead costs for HVDC transmission lines, while ``c`` uses more accurate costs distinguishing between overhead and underwater sections and including inverter pairs). The setting ``v1.25`` will limit the total volume of line expansion to 25% of currently installed capacities weighted by individual line lengths. The setting ``c1.25`` will allow to build a transmission network that costs no more than 25 % more than the current system." diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index f13221f477..60810405e8 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1292,7 +1292,7 @@ def attach_stores( tech_map = { key: value for key, value in estimate_renewable_caps["technology_mapping"].items() - if value not in tyndp_renewable_carriers + if key in estimate_renewable_caps["technologies"] } expansion_limit = estimate_renewable_caps["expansion_limit"] year = estimate_renewable_caps["year"] From 5e8d6197f3b5c1ed1d81a75d8835d2354ed4f4a5 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 3 Jul 2025 14:43:45 +0200 Subject: [PATCH 48/49] feat: split up tyndp renewable carriers into separate list --- config/config.default.yaml | 1 + config/config.tyndp.yaml | 3 ++- config/test/config.tyndp.yaml | 3 ++- doc/configtables/electricity.csv | 5 +++-- rules/build_electricity.smk | 16 ---------------- rules/build_sector.smk | 1 - rules/solve_myopic.smk | 11 +++++++---- rules/solve_perfect.smk | 3 +++ scripts/_helpers.py | 25 ------------------------- scripts/add_brownfield.py | 5 +---- scripts/add_electricity.py | 11 +++-------- scripts/add_existing_baseyear.py | 5 +---- 12 files changed, 23 insertions(+), 66 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 7a9db61877..16343a5c39 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -112,6 +112,7 @@ electricity: conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float, hydro] + tyndp_renewable_carriers: [] pecd_renewable_profiles: enable: false diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 8eb93c8985..45c07636fd 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -40,7 +40,8 @@ co2_budget: electricity: base_network: tyndp-raw - renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro] + renewable_carriers: [solar, solar-hsat, onwind, hydro] + tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 5b2acbdd19..9e781ec6e5 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -44,7 +44,8 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + renewable_carriers: [solar, solar-hsat, onwind] + tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 72f8712196..2e8a61e495 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -30,11 +30,12 @@ everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignit conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound." ,,, renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. +tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv-rooftop, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh}",List of TYNDP renewable generators to include in the model. Technologies covered by specified TYNDP renewable carriers need to be removed from `estimate_renewable_carriers` technology list. pecd_renewable_profiles,,, -- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. -- technologies,,, --- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tydnp_renewable_carriers``. --- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. Carriers not listed use the default renewable profiles. +-- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. +-- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 455a7762af..0797d3aef8 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -601,7 +601,6 @@ def input_class_regions(w): ) for tech in set(config_provider("electricity", "renewable_carriers")(w)) - {"hydro"} - - set(tyndp_renewable_carriers(w)) } @@ -786,20 +785,6 @@ rule cluster_network: "../scripts/cluster_network.py" -def tyndp_renewable_carriers(w): - return ( - list( - chain( - *config_provider( - "electricity", "pecd_renewable_profiles", "technologies" - )(w).values() - ) - ) - if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) - else [] - ) - - def input_profile_tech(w): return { f"profile_{tech}": resources( @@ -808,7 +793,6 @@ def input_profile_tech(w): else f"profile_{tech}.nc" ) for tech in set(config_provider("electricity", "renewable_carriers")(w)) - - set(tyndp_renewable_carriers(w)) } diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1a9605850e..3b1569cf59 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1228,7 +1228,6 @@ def input_profile_offwind(w): f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") for tech in ["offwind-ac", "offwind-dc", "offwind-float"] if (tech in config_provider("electricity", "renewable_carriers")(w)) - and (tech not in tyndp_renewable_carriers(w)) } diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index e880b962ac..55b4f097d1 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -10,6 +10,9 @@ rule add_existing_baseyear: pecd_renewable_profiles=config_provider( "electricity", "pecd_renewable_profiles" ), + tyndp_renewable_carriers=config_provider( + "electricity", "tyndp_renewable_carriers" + ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), @@ -63,10 +66,7 @@ rule add_existing_baseyear: def input_profile_tech_brownfield(w): return { f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") - for tech in ( - set(config_provider("electricity", "renewable_carriers")(w)) - - set(tyndp_renewable_carriers(w)) - ) + for tech in (set(config_provider("electricity", "renewable_carriers")(w))) if tech != "hydro" } @@ -89,6 +89,9 @@ rule add_brownfield: pecd_renewable_profiles=config_provider( "electricity", "pecd_renewable_profiles" ), + tyndp_renewable_carriers=config_provider( + "electricity", "tyndp_renewable_carriers" + ), drop_leap_day=config_provider("enable", "drop_leap_day"), carriers=config_provider("electricity", "renewable_carriers"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 0b74cb35d1..f8792dc88c 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -8,6 +8,9 @@ rule add_existing_baseyear: pecd_renewable_profiles=config_provider( "electricity", "pecd_renewable_profiles" ), + tyndp_renewable_carriers=config_provider( + "electricity", "tyndp_renewable_carriers" + ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), diff --git a/scripts/_helpers.py b/scripts/_helpers.py index c0aa7ef24e..9543a25106 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -10,7 +10,6 @@ import re import time from functools import partial, wraps -from itertools import chain from pathlib import Path from tempfile import NamedTemporaryFile from typing import Callable, Union @@ -1128,27 +1127,3 @@ def extract_grid_data_tyndp( h2_grid.index = h2_grid.apply(make_index, axis=1, args=(carrier,)) return h2_grid - - -def get_tyndp_res_carriers(pecd_renewable_profiles: dict): - """ - Function to return all TYNDP renewable carriers specified in the configuration file for PECD profiles. - - The function makes sure TYNDP renewable carriers are only returned if PECD profiles are enabled. - - Parameters - ---------- - pecd_renewable_profiles : dict - Dictionary that contains all TYNDP renewable carriers of the PECD profiles. - - Returns - ------- - tyndp_renewable_carriers : list - List of TYNDP renewable carriers. - """ - tyndp_renewable_carriers = ( - list(chain(*pecd_renewable_profiles["technologies"].values())) - if pecd_renewable_profiles["enable"] - else [] - ) - return tyndp_renewable_carriers diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 17e9ee75ae..03771db9aa 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -15,7 +15,6 @@ from scripts._helpers import ( configure_logging, get_snapshots, - get_tyndp_res_carriers, sanitize_custom_columns, set_scenario_config, update_config_from_wildcards, @@ -357,9 +356,7 @@ def update_dynamic_ptes_capacity( n = pypsa.Network(snakemake.input.network) - tyndp_renewable_carriers = get_tyndp_res_carriers( - snakemake.params.pecd_renewable_profiles - ) + tyndp_renewable_carriers = snakemake.params.tyndp_renewable_carriers adjust_renewable_profiles( n, snakemake.input, snakemake.params, year, tyndp_renewable_carriers diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 60810405e8..a52a72dd58 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -64,7 +64,6 @@ PYPSA_V1, configure_logging, get_snapshots, - get_tyndp_res_carriers, rename_techs, set_scenario_config, update_p_nom_max, @@ -1216,16 +1215,12 @@ def attach_stores( params.link_length_factor, ) - tyndp_renewable_carriers = get_tyndp_res_carriers( - params.electricity["pecd_renewable_profiles"] - ) + tyndp_renewable_carriers = params.electricity["tyndp_renewable_carriers"] if len(tyndp_renewable_carriers) > 0: logger.info( - f"Skipping renewable carriers - they will be attached later with TYNDP data: {', '.join(tyndp_renewable_carriers)}" + f"Skipping TYNDP renewable carriers - they will be attached later with TYNDP data: {', '.join(tyndp_renewable_carriers)}" ) - renewable_carriers = set(params.electricity["renewable_carriers"]).difference( - tyndp_renewable_carriers - ) + renewable_carriers = params.electricity["renewable_carriers"] extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 5ffd2f0f81..1eae4a215d 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -19,7 +19,6 @@ from scripts._helpers import ( configure_logging, - get_tyndp_res_carriers, sanitize_custom_columns, set_scenario_config, update_config_from_wildcards, @@ -761,9 +760,7 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector renewable_carriers = snakemake.params.carriers - tyndp_renewable_carriers = get_tyndp_res_carriers( - snakemake.params.pecd_renewable_profiles - ) + tyndp_renewable_carriers = snakemake.params.tyndp_renewable_carriers baseyear = snakemake.params.baseyear From f191c287b8caf22246d125c553430ab96ece1718 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 3 Jul 2025 14:50:22 +0200 Subject: [PATCH 49/49] refactor: consistent naming convention of pecd_techs and tyndp_renewable_carriers --- rules/build_sector.smk | 4 ++-- rules/solve_myopic.smk | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 3b1569cf59..1a0a49872d 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1231,7 +1231,7 @@ def input_profile_offwind(w): } -pecd_renewable_profiles = branch( +pecd_techs = branch( config_provider("electricity", "pecd_renewable_profiles", "enable"), config_provider("electricity", "pecd_renewable_profiles", "technologies"), ) @@ -1240,7 +1240,7 @@ pecd_renewable_profiles = branch( def input_profile_pecd(w): return { f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") - for tech in pecd_renewable_profiles(w) + for tech in pecd_techs(w) } diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 55b4f097d1..473e274520 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -74,7 +74,7 @@ def input_profile_tech_brownfield(w): def input_profile_tech_brownfield_pecd(w): return { f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") - for tech in pecd_renewable_profiles(w) + for tech in pecd_techs(w) }