From 1cf34a9b730976949ac3cb604bdb4414a491a664 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 23 May 2025 17:39:38 +0200 Subject: [PATCH 001/165] feat: skip tyndp_renewable_profile technologies in workflow and build_renewable_profiles --- config/config.default.yaml | 7 +++++++ config/config.tyndp.yaml | 6 ++++++ config/test/config.tyndp.yaml | 6 ++++++ rules/build_electricity.smk | 12 +++++++++++- rules/build_sector.smk | 1 + rules/solve_myopic.smk | 5 ++++- scripts/add_electricity.py | 23 +++++++++++++++++++++-- 7 files changed, 56 insertions(+), 4 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index da6c69d146..234b0ecabc 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -111,6 +111,13 @@ electricity: conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float, hydro] + tyndp_renewable_profiles: + enable: false + technologies: + - offwind-ac + - offwind-dc + - offwind-float + estimate_renewable_capacities: enable: true from_gem: true diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index e151f88b43..8a0836ec6c 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -38,6 +38,12 @@ co2_budget: electricity: base_network: tyndp-raw transmission_limit: v1.0 + tyndp_renewable_profiles: + enable: true + technologies: + - offwind-ac + - offwind-dc + - offwind-float links: p_max_pu: 1.0 diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 77f3498685..4cbd874a44 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -35,6 +35,12 @@ co2_budget: electricity: base_network: tyndp-raw transmission_limit: v1.0 + tyndp_renewable_profiles: + enable: true + technologies: + - offwind-ac + - offwind-dc + - offwind-float extendable_carriers: Generator: [OCGT] diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 673b07a770..5966f7d2ee 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -540,6 +540,7 @@ def input_class_regions(w): ) for tech in set(config_provider("electricity", "renewable_carriers")(w)) - {"hydro"} + - set(tyndp_renewable_profiles(w)) } @@ -705,6 +706,14 @@ rule cluster_network: "../scripts/cluster_network.py" +def tyndp_renewable_profiles(w): + return ( + config_provider("electricity", "tyndp_renewable_profiles", "technologies")(w) + if config_provider("electricity", "tyndp_renewable_profiles", "enable")(w) + else [] + ) + + def input_profile_tech(w): return { f"profile_{tech}": resources( @@ -712,7 +721,8 @@ def input_profile_tech(w): if tech != "hydro" else f"profile_{tech}.nc" ) - for tech in config_provider("electricity", "renewable_carriers")(w) + for tech in set(config_provider("electricity", "renewable_carriers")(w)) + - set(tyndp_renewable_profiles(w)) } diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 3dbc4afd0f..b56071a4df 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1226,6 +1226,7 @@ def input_profile_offwind(w): f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") for tech in ["offwind-ac", "offwind-dc", "offwind-float"] if (tech in config_provider("electricity", "renewable_carriers")(w)) + and (tech not in tyndp_renewable_profiles(w)) } diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 87379b08b6..e2068bc817 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -59,7 +59,10 @@ rule add_existing_baseyear: def input_profile_tech_brownfield(w): return { f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") - for tech in config_provider("electricity", "renewable_carriers")(w) + for tech in ( + set(config_provider("electricity", "renewable_carriers")(w)) + - set(tyndp_renewable_profiles(w)) + ) if tech != "hydro" } diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 0a64e364a8..d7c9a314cb 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1203,7 +1203,22 @@ def attach_stores( params.link_length_factor, ) - renewable_carriers = set(params.electricity["renewable_carriers"]) + tyndp_renewable_profiles = ( + params.electricity["tyndp_renewable_profiles"]["technologies"] + if params.electricity["tyndp_renewable_profiles"]["enable"] + else [] + ) + if len(tyndp_renewable_profiles) > 0: + logger.info( + f"Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'. They will be attached later on with TYNDP data." + ) + renewable_carriers = set( + [ + carrier + for carrier in params.electricity["renewable_carriers"] + if carrier not in tyndp_renewable_profiles + ] + ) extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { @@ -1267,7 +1282,11 @@ def attach_stores( "in rule `add_existing_baseyear` with foresight mode 'myopic'." ) else: - tech_map = estimate_renewable_caps["technology_mapping"] + tech_map = { + key: value + for key, value in estimate_renewable_caps["technology_mapping"].items() + if value not in tyndp_renewable_profiles + } expansion_limit = estimate_renewable_caps["expansion_limit"] year = estimate_renewable_caps["year"] From 3be60a1be9fa1b456a2a27f6ca7dfbbc90b8f0f2 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 14:34:02 +0200 Subject: [PATCH 002/165] feat: clean pecd data for years 2030, 2040, 2050 and build_renewable_profiles_pecd --- rules/build_electricity.smk | 70 ++++++++++++ scripts/build_renewable_profiles_pecd.py | 93 ++++++++++++++++ scripts/clean_pecd_data.py | 135 +++++++++++++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 scripts/build_renewable_profiles_pecd.py create mode 100644 scripts/clean_pecd_data.py diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 5966f7d2ee..614cb0e112 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -381,6 +381,76 @@ rule build_renewable_profiles: "../scripts/build_renewable_profiles.py" +def input_data_pecd(w): + return { + f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + pyear + ".csv") + for pyear in [ + "2030", + "2040", + "2050", + ] # complete PECD data is available for the years 2030, 2040 + } + + +rule build_renewable_profiles_pecd: + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + renewable=config_provider("renewable"), + planning_horizons=config_provider("scenario", "planning_horizons"), + input: + unpack(input_data_pecd), + availability_matrix=resources("availability_matrix_{clusters}_{technology}.nc"), + offshore_shapes=resources("offshore_shapes.geojson"), + distance_regions=resources("regions_onshore_base_s_{clusters}.geojson"), + resource_regions=lambda w: ( + resources("regions_onshore_base_s_{clusters}.geojson") + if w.technology in ("onwind", "solar", "solar-hsat") + else resources("regions_offshore_base_s_{clusters}.geojson") + ), + cutout=lambda w: input_cutout( + w, config_provider("renewable", w.technology, "cutout")(w) + ), + output: + profile=resources("profile_pecd_{clusters}_{technology}.nc"), + log: + logs("build_renewable_profile_pecd_{clusters}_{technology}.log"), + benchmark: + benchmarks("build_renewable_profile_pecd_{clusters}_{technology}") + threads: 1 + resources: + mem_mb=4000, + wildcard_constraints: + technology="(?!hydro).*", # Any technology other than hydro + conda: + "../envs/environment.yaml" + script: + "../scripts/build_renewable_profiles_pecd.py" + + +rule clean_pecd_data: + params: + scenario=config_provider("tyndp_scenario"), + snapshots=config_provider("snapshots"), + input: + offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", + onshore_buses=resources("busmap_base_s_all.csv"), + fn_pecd="/Users/daniel/Desktop/Work/OET/Projects/open-tyndp/data/2024/20250313_ENTSO-E_ENTSOG_TYNDP_2024_Scenarios_Inputs/PECD/{planning_horizons}/", + output: + pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), + log: + logs("clean_pecd_data_{technology}_{planning_horizons}.log"), + benchmark: + benchmarks("clean_pecd_data_{technology}_{planning_horizons}") + threads: 4 + resources: + mem_mb=4000, + conda: + "../envs/environment.yaml" + script: + "../scripts/clean_pecd_data.py" + + rule build_monthly_prices: input: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py new file mode 100644 index 0000000000..3d28633ae8 --- /dev/null +++ b/scripts/build_renewable_profiles_pecd.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# +# SPDX-License-Identifier: MIT +""" +Creates renewable profiles for each region from PECD and TYNDP input data containing the available +generation time series (based on PECD weather data) from the node for onshore wind, AC-connected offshore wind, +DC-connected offshore wind and solar PV generators. + +.. note:: Hydroelectric profiles will be built in script :mod:`build_hydro_profiles_PECD`. Not yet implemented. + +Outputs +------- + +- ``resources/profile_pecd_{clusters}_{technology}.nc`` with the following structure + + =================== ==================== ========================================================= + Field Dimensions Description + =================== ==================== ========================================================= + profile year, bus, bin, time the per unit hourly availability factors for each bus + =================== ==================== ========================================================= +""" + +import logging + +import numpy as np +import pandas as pd +import xarray as xr + +from scripts._helpers import ( + configure_logging, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake( + "build_renewable_profiles_pecd", + clusters="all", + technology="offwind-ac", + ) + configure_logging(snakemake) + set_scenario_config(snakemake) + + technology = snakemake.wildcards.technology + pyears = snakemake.params.planning_horizons + + profiles = [] + + for year in pyears: + logger.info( + f"Extract PECD capacity factor time series for year {year} for technology {technology}..." + ) + + if int(year) not in [2030, 2040, 2050]: + year = np.clip(10 * (year // 10), 2030, 2050) + logger.warning( + "Planning horizon doesn't match available TYNDP data. " + f"Falling back to previous available year {year}." + ) + + profile = ( + pd.read_csv( + snakemake.input[f"pecd_data_{year}"], parse_dates=True, index_col=0 + ) + .rename_axis("time") + .reset_index() + .melt(id_vars=["time"], var_name="bus", value_name="profile") + .assign(bin=0, year=year) + .set_index(["time", "bus", "bin", "year"]) + .to_xarray() + ) + + profiles.append(profile) + + ds = xr.merge(profiles) + + # TODO: Later on the max capacities for renewable technologies and regions can be added here + # profiles = xr.merge(profiles) + # p_nom_max = place_holder + # average_distance = place_holder + # ds = xr.merge( + # [ + # profiles, + # p_nom_max.rename("p_nom_max"), + # average_distance.rename("average_distance"), + # ] + # ) + + ds.to_netcdf(snakemake.output.profile) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py new file mode 100644 index 0000000000..a14c5c3b45 --- /dev/null +++ b/scripts/clean_pecd_data.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# +# SPDX-License-Identifier: MIT +""" +Loads and cleans the available PECD capacity factor generation time series based on PECD weather data. +The script is executed for a given technology, and planning horizon. Technologies can be one of: + + * CSP_noStorage, + * CSP_withStorage, + * LFSolarPV, + * LFSolarPVRooftop, + * LFSolarPVUtility, + * Wind_Offshore, + * Wind_Onshore. + +Outputs +------- +Cleaned csv file with capacity factor generation time series and regions as columns. +""" + +import logging +import multiprocessing as mp +import os +from functools import partial +from pathlib import Path + +import pandas as pd +from tqdm import tqdm + +from scripts._helpers import ( + configure_logging, + get_snapshots, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + + +def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: str): + fn = Path(fn_pecd, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + + if not os.path.isfile(fn): + return None + + pecd_bus = pd.read_csv( + fn, + skiprows=10, + usecols=lambda name: name == "Date" or name == "Hour" or name == str(cyear), + ) + datetime_str = f"{cyear}." + pecd_bus["Date"].str.cat( + (pecd_bus["Hour"] - 1).astype(str), sep=" " + ) + cf_pecd = ( + pecd_bus.set_index(pd.to_datetime(datetime_str, format="%Y.%d.%m. %H")) + .drop(columns=["Date", "Hour"]) + .rename(columns={str(cyear): node}) + ) + + return cf_pecd + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake( + "clean_pecd_data", + clusters="all", + technology="offwind-ac", + planning_horizons=2030, + ) + configure_logging(snakemake) + set_scenario_config(snakemake) + + # Climate year from snapshots + cyear = get_snapshots(snakemake.params.snapshots)[0].year + if int(cyear) < 1982 or int(cyear) > 2019: + # TODO: Note that because of this fallback, the snapshots of the profiles will not always match with the model snapshots + logger.warning( + "Snapshot year doesn't match available TYNDP data. Falling back to 2009." + ) + cyear = 2009 + + # Planning year + pyear = str(snakemake.wildcards.planning_horizons) + + # Technology as in PECD terminology + # TODO: find solution for solar profiles being differentiated between Utility and Rooftop for Italy + pecd_tech_dict = { + "offwind-ac": "Wind_Offshore", + "offwind-dc": "Wind_Offshore", + "offwind-float": "Wind_Offshore", + "onwind": "Wind_Onshore", + "solar": "LFSolarPV", + "solar-hsat": "LFSolarPV", + } + pecd_tech = pecd_tech_dict[snakemake.wildcards.technology] + + offshore_buses = pd.read_excel(snakemake.input.offshore_buses, index_col=0) + onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) + + nodes = ( + offshore_buses.index if pecd_tech == "Wind_Offshore" else onshore_buses.index + ) + fn_pecd = snakemake.input.fn_pecd + + if pyear == "2050": + logger.warning( + "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." + ) + pyear = "2040" + fn_pecd = fn_pecd.replace("2050", "2040") + + # Load and prep electricity demand + tqdm_kwargs = { + "ascii": False, + "unit": " nodes", + "total": len(nodes), + "desc": "Loading PECD capacity factor data", + } + + func = partial( + read_pecd_file, + fn_pecd=fn_pecd, + cyear=cyear, + pyear=pyear, + technology=pecd_tech, + ) + + with mp.Pool(processes=snakemake.threads) as pool: + demand = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + + pecd_df = pd.concat(demand, axis=1) + + pecd_df.to_csv(snakemake.output.pecd_data_clean) From 9f0b674f88ef05e28d052e77d341fc0538ca6163 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 14:35:09 +0200 Subject: [PATCH 003/165] feat: add pecd profiles as input to prepare_sector_network and add_brownfield rules --- rules/build_sector.smk | 8 ++++++++ rules/solve_myopic.smk | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index b56071a4df..8088ee2262 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1230,6 +1230,13 @@ def input_profile_offwind(w): } +def input_profile_pecd(w): + return { + f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") + for tech in tyndp_renewable_profiles(w) + } + + rule build_egs_potentials: params: snapshots=config_provider("snapshots"), @@ -1380,6 +1387,7 @@ rule prepare_sector_network: scaling_factor=config_provider("load", "scaling_factor"), input: unpack(input_profile_offwind), + unpack(input_profile_pecd), unpack(input_heat_source_power), **rules.cluster_gas_network.output, **rules.build_gas_input_locations.output, diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index e2068bc817..a4c4dfdf9e 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -67,6 +67,13 @@ def input_profile_tech_brownfield(w): } +def input_profile_tech_brownfied_pecd(w): + return { + f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") + for tech in tyndp_renewable_profiles(w) + } + + rule add_brownfield: params: H2_retrofit=config_provider("sector", "H2_retrofit"), @@ -84,6 +91,7 @@ rule add_brownfield: ), input: unpack(input_profile_tech_brownfield), + unpack(input_profile_tech_brownfied_pecd), simplify_busmap=resources("busmap_base_s.csv"), cluster_busmap=resources("busmap_base_s_{clusters}.csv"), network=resources( From c0ce59fe7cf933fad5ff6ee150d2ef3401e14d8a Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 15:56:36 +0200 Subject: [PATCH 004/165] fix: add hotfix for add_existing_baseyear existing renewable capacities and fix name of profiles in add_brownfield rule --- rules/solve_myopic.smk | 3 ++- scripts/add_existing_baseyear.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index a4c4dfdf9e..7b6eb06a15 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -7,6 +7,7 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), + electricity=config_provider("electricity"), existing_capacities=config_provider("existing_capacities"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), @@ -69,7 +70,7 @@ def input_profile_tech_brownfield(w): def input_profile_tech_brownfied_pecd(w): return { - f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") + f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") for tech in tyndp_renewable_profiles(w) } diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index f49bd548b9..50d098133e 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -89,6 +89,13 @@ def add_existing_renewables( Modifies df_agg in-place """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} + # TODO: remove when TYNDP renewable generators are added + if len(tyndp_renewable_profiles) > 0: + logger.info( + f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'." + ) + for k in tyndp_renewable_profiles: + tech_map.pop(k, None) irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() irena = irena.query("Country in @countries") @@ -726,7 +733,11 @@ def add_heating_capacities_installed_before_baseyear( update_config_from_wildcards(snakemake.config, snakemake.wildcards) options = snakemake.params.sector - + tyndp_renewable_profiles = ( + snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] + if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] + else [] + ) baseyear = snakemake.params.baseyear n = pypsa.Network(snakemake.input.network) From cabc766afe04f4244162d33a94857de64eb20002 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 16:09:58 +0200 Subject: [PATCH 005/165] fix: fix pylint --- scripts/add_existing_baseyear.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 50d098133e..7c913b876c 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -90,6 +90,11 @@ def add_existing_renewables( """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} # TODO: remove when TYNDP renewable generators are added + tyndp_renewable_profiles = ( + snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] + if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] + else [] + ) if len(tyndp_renewable_profiles) > 0: logger.info( f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'." @@ -733,11 +738,7 @@ def add_heating_capacities_installed_before_baseyear( update_config_from_wildcards(snakemake.config, snakemake.wildcards) options = snakemake.params.sector - tyndp_renewable_profiles = ( - snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] - if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] - else [] - ) + baseyear = snakemake.params.baseyear n = pypsa.Network(snakemake.input.network) From 686df432e2a8c308fa16a9ce7b0111e927d68b25 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 16:20:14 +0200 Subject: [PATCH 006/165] fix: fix pylint better --- scripts/add_existing_baseyear.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 7c913b876c..d2bf9aac1b 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -68,6 +68,7 @@ def add_existing_renewables( costs: pd.DataFrame, df_agg: pd.DataFrame, countries: list[str], + tyndp_renewable_profiles: list[str], ) -> None: """ Add existing renewable capacities to conventional power plant data. @@ -82,6 +83,8 @@ def add_existing_renewables( Network containing topology and generator data countries : list List of country codes to consider + tyndp_renewable_profiles: list + List of renewable profile technologies taken from tyndp PECD Returns ------- @@ -90,11 +93,6 @@ def add_existing_renewables( """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} # TODO: remove when TYNDP renewable generators are added - tyndp_renewable_profiles = ( - snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] - if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] - else [] - ) if len(tyndp_renewable_profiles) > 0: logger.info( f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'." @@ -161,6 +159,7 @@ def add_power_capacities_installed_before_baseyear( countries: list[str], capacity_threshold: float, lifetime_values: dict[str, float], + tyndp_renewable_profiles: list[str], ) -> None: """ Add power generation capacities installed before base year. @@ -183,6 +182,8 @@ def add_power_capacities_installed_before_baseyear( Minimum capacity threshold lifetime_values : dict Default values for missing data + tyndp_renewable_profiles: list + List of renewable profile technologies taken from tyndp PECD """ logger.debug(f"Adding power capacities installed before {baseyear}") @@ -235,6 +236,7 @@ def add_power_capacities_installed_before_baseyear( costs=costs, n=n, countries=countries, + tyndp_renewable_profiles=tyndp_renewable_profiles, ) # drop assets which are already phased out / decommissioned phased_out = df_agg[df_agg["DateOut"] < baseyear].index @@ -739,6 +741,12 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector + tyndp_renewable_profiles = ( + snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] + if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] + else [] + ) + baseyear = snakemake.params.baseyear n = pypsa.Network(snakemake.input.network) @@ -765,6 +773,7 @@ def add_heating_capacities_installed_before_baseyear( countries=snakemake.config["countries"], capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"], lifetime_values=snakemake.params.costs["fill_values"], + tyndp_renewable_profiles=tyndp_renewable_profiles, ) if options["heating"]: From 542668d0610b65080d25bddf3de7d1a888c4e093 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 23 May 2025 15:01:09 +0200 Subject: [PATCH 007/165] feat: introduce clean_tyndp_offshore_hubs --- config/config.default.yaml | 4 + rules/build_sector.smk | 15 +++ rules/retrieve.smk | 2 + scripts/clean_tyndp_offshore_hubs.py | 142 +++++++++++++++++++++++++++ 4 files changed, 163 insertions(+) create mode 100644 scripts/clean_tyndp_offshore_hubs.py diff --git a/config/config.default.yaml b/config/config.default.yaml index 94aaee2637..157e3ed359 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -797,6 +797,10 @@ sector: methanol: 121 gas: 122 oil: 125 + offshore_hubs: + enable: false + tyndp_scenario: '' # NT, DE or GA + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry industry: diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 388a077049..4ea187cd3d 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1339,6 +1339,21 @@ if config["sector"]["h2_topology_tyndp"]: "../scripts/build_tyndp_h2_imports.py" +# TODO Set the right if +if True: + + rule clean_tyndp_offshore_hubs: + params: + planning_horizons=config_provider("scenario", "planning_horizons"), + # TODO Select the right scenario + scenario=config_provider("load", "tyndp_scenario"), + input: + nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), + grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), + script: + "../scripts/clean_tyndp_offshore_hubs.py" + + rule prepare_sector_network: params: time_resolution=config_provider("clustering", "temporal", "resolution_sector"), diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4f2b4646de..c325020b04 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -179,6 +179,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" h2_reference_grid="data/tyndp_2024_bundle/Line data/ReferenceGrid_Hydrogen.xlsx", electricity_demand=directory("data/tyndp_2024_bundle/Demand Profiles"), h2_imports="data/tyndp_2024_bundle/Hydrogen/H2 IMPORTS GENERATORS PROPERTIES.xlsx", + offshore_nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), + offshore_grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), log: "logs/retrieve_tyndp_bundle.log", retries: 2 diff --git a/scripts/clean_tyndp_offshore_hubs.py b/scripts/clean_tyndp_offshore_hubs.py new file mode 100644 index 0000000000..de9bfbd60d --- /dev/null +++ b/scripts/clean_tyndp_offshore_hubs.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: : 2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +This script is used to clean TYNDP Scenario Building offshore hubs data to be used in the PyPSA-Eur workflow. Depending on the scenario, different planning years (`pyear`) are available. DE and GA are defined for 2030, 2040 and 2050. NT scenario is only defined for 2030 and 2040. All the planning years are read at once. +""" + +import logging + +import geopandas as gpd +import pandas as pd +from _helpers import configure_logging, set_scenario_config +from shapely.geometry import Point + +logger = logging.getLogger(__name__) + +GEO_CRS = "EPSG:4326" + + +def load_offshore_hubs(fn: str): + """ + Load offshore hubs coordinates and format data. + """ + column_dict = { + "OFFSHORE_NODE": "Bus", + "OFFSHORE_NODE_TYPE": "type", + "HOME_NODE": "location", + "LAT": "y", + "LON": "x", + } + + nodes = pd.read_excel( + fn, + sheet_name="NODE", + ).rename(columns=column_dict) + + nodes["geometry"] = nodes.apply(lambda row: Point(row["x"], row["y"]), axis=1) + nodes = gpd.GeoDataFrame(nodes, geometry="geometry", crs=GEO_CRS) + + # rename UK in GB + nodes[["Bus", "location"]] = nodes[["Bus", "location"]].replace("UK", "GB") + + return nodes + + +def expand_all_scenario(df: pd.DataFrame, scenarios: list): + all_mask = df["scenario"] == "All" + all_rows = ( + df[all_mask] + .drop(columns="scenario") + .merge(pd.DataFrame({"scenario": scenarios}), how="cross") + ) + return pd.concat([df[~all_mask], all_rows], ignore_index=True) + + +def load_offshore_grid( + fn: str, nodes: pd.DataFrame, scenario: str, planning_horizons: list[int] +): + """ + Load offshore grid (electricity and hydrogen) and format data. + """ + column_dict = { + "FROM": "bus0", + "TO": "bus1", + "YEAR": "pyear", + "SCENARIO": "scenario", + "MARKET": "carrier", + "CAPACITY": "p_nom", + "CAPEX": "capital_cost", + "OPEX": "marginal_cost", + } + + scenario_dict = { + "Distributed Energy": "DE", + "Global Ambition": "GA", + "National Trends": "NT", + } + + # Load reference grid + grid = ( + pd.read_excel( + fn, + sheet_name="Reference grid", + ) + .rename(columns=column_dict) + .assign( + p_min_pu=0, + p_max_pu=1, + ) + ) + grid["carrier"] = grid["carrier"].replace("E", "DC") + grid = expand_all_scenario(grid, scenario_dict.values()).query( + "scenario == @scenario" + ) + + # Load costs data + grid_costs = ( + pd.read_excel( + fn, + sheet_name="COST", + ) + .rename(columns=column_dict) + .query("pyear in @planning_horizons") + ) + grid_costs["carrier"] = grid_costs["carrier"].replace("E", "DC") + grid_costs["scenario"] = grid_costs["scenario"].replace(scenario_dict) + + # Rename UK in GB + grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB") + grid_costs[["bus0", "bus1"]] = grid_costs[["bus0", "bus1"]].replace("UK", "GB") + + # Merge information + grid = grid.merge( + grid_costs, how="left", on=["bus0", "bus1", "pyear", "scenario", "carrier"] + ) + + return grid + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "clean_tyndp_offshore_hubs", configfiles="config/test/config.tyndp.yaml" + ) + + configure_logging(snakemake) + set_scenario_config(snakemake) + + # Parameters + scenario = snakemake.params["scenario"] + planning_horizons = snakemake.params["planning_horizons"] + + nodes = load_offshore_hubs(snakemake.input.nodes) + + grid = load_offshore_grid( + snakemake.input.grid, nodes, snakemake.params["scenario"], planning_horizons + ) + + # Save prepped electricity demand + nodes.to_csv(snakemake.output.electricity_demand_prepped) From 66d69d6116aec6860c06184023200990bc94a64c Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 27 May 2025 16:31:55 +0200 Subject: [PATCH 008/165] doc: document newly added functions --- scripts/clean_tyndp_offshore_hubs.py | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scripts/clean_tyndp_offshore_hubs.py b/scripts/clean_tyndp_offshore_hubs.py index de9bfbd60d..bcd2a8bc6e 100644 --- a/scripts/clean_tyndp_offshore_hubs.py +++ b/scripts/clean_tyndp_offshore_hubs.py @@ -20,6 +20,18 @@ def load_offshore_hubs(fn: str): """ Load offshore hubs coordinates and format data. + + Parameters + ---------- + fn : str + Path to the Excel file containing offshore hub data. + + Returns + ------- + gpd.GeoDataFrame + GeoDataFrame containing the offshore hub data. + + The GeoDataFrame uses the coordinate reference system defined by `GEO_CRS`. """ column_dict = { "OFFSHORE_NODE": "Bus", @@ -58,6 +70,25 @@ def load_offshore_grid( ): """ Load offshore grid (electricity and hydrogen) and format data. + + Parameters + ---------- + fn : str + Path to the Excel file containing offshore grid data. + nodes : pd.DataFrame + DataFrame containing node information (currently not used in function body + but may be needed for validation or future functionality). + scenario : str + Scenario identifier to filter the grid data. Must be one of the scenario + codes: "DE" (Distributed Energy), "GA" (Global Ambition), or + "NT" (National Trends). + planning_horizons : list[int] + List of planning years to include in the cost data filtering. + + Returns + ------- + pd.DataFrame + DataFrame containing the merged offshore grid data. """ column_dict = { "FROM": "bus0", From 85dabffef114b988e8f6bcbe368e4885cab67048 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 27 May 2025 16:32:35 +0200 Subject: [PATCH 009/165] feat: improve configuration based on master --- config/config.default.yaml | 4 +--- rules/build_sector.smk | 7 +------ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 157e3ed359..9c38bbbca8 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -797,9 +797,7 @@ sector: methanol: 121 gas: 122 oil: 125 - offshore_hubs: - enable: false - tyndp_scenario: '' # NT, DE or GA + offshore_hubs: false # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 4ea187cd3d..42f526479d 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1338,15 +1338,10 @@ if config["sector"]["h2_topology_tyndp"]: script: "../scripts/build_tyndp_h2_imports.py" - -# TODO Set the right if -if True: - rule clean_tyndp_offshore_hubs: params: planning_horizons=config_provider("scenario", "planning_horizons"), - # TODO Select the right scenario - scenario=config_provider("load", "tyndp_scenario"), + scenario=config_provider("tyndp_scenario"), input: nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), From eaefe8ca35cdfb084d247f8b1a1c959457a6d66b Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 27 May 2025 16:38:59 +0200 Subject: [PATCH 010/165] feat: define outputs --- rules/build_sector.smk | 3 +++ scripts/clean_tyndp_offshore_hubs.py | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 42f526479d..e0dadf4401 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1345,6 +1345,9 @@ if config["sector"]["h2_topology_tyndp"]: input: nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), + output: + buses=resources("offshore_buses.csv"), + offshore_grid=resources("offshore_grid.csv"), script: "../scripts/clean_tyndp_offshore_hubs.py" diff --git a/scripts/clean_tyndp_offshore_hubs.py b/scripts/clean_tyndp_offshore_hubs.py index bcd2a8bc6e..0e25024684 100644 --- a/scripts/clean_tyndp_offshore_hubs.py +++ b/scripts/clean_tyndp_offshore_hubs.py @@ -169,5 +169,6 @@ def load_offshore_grid( snakemake.input.grid, nodes, snakemake.params["scenario"], planning_horizons ) - # Save prepped electricity demand - nodes.to_csv(snakemake.output.electricity_demand_prepped) + # Save data + nodes.to_csv(snakemake.output.buses, index=False) + grid.to_csv(snakemake.output.offshore_grid, index=False) From 8491dd2879f602085c5bbb4ae235eb3b72196e4e Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 27 May 2025 16:42:52 +0200 Subject: [PATCH 011/165] feat: enable offshore hubs by default for tyndp configurations --- config/config.tyndp.yaml | 1 + config/test/config.tyndp.yaml | 1 + doc/configtables/sector.csv | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 3a4578f977..49377a9a5e 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -125,6 +125,7 @@ sector: enable: true carriers: - H2 + offshore_hubs: true clustering: mode: administrative # TODO Switch to Bidding zones to preserve bidding zones shapes diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index cd536710fb..09f041c9e6 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -144,6 +144,7 @@ sector: enable: true carriers: - H2 + offshore_hubs: true clustering: mode: administrative # TODO Switch to Bidding zones to preserve bidding zones shapes diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv index db4f99b172..7a62724461 100644 --- a/doc/configtables/sector.csv +++ b/doc/configtables/sector.csv @@ -229,4 +229,5 @@ imports,,, -- limit,TWh,float,Maximum allowed renewable energy imports -- limit_sense,--,"{==, <=, >=}",Sense of the limit -- price,,"{H2, NH3, methanol, gas, oil}", --- -- {carrier},currency/MWh,float,Price for importing renewable energy of carrier \ No newline at end of file +-- -- {carrier},currency/MWh,float,Price for importing renewable energy of carrier +offshore_hubs,--,"{true, false}",Add option for TYNDP offshore hubs \ No newline at end of file From 4162538b129a343ee45d30d6fbe161afe0502346 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 27 May 2025 17:58:32 +0200 Subject: [PATCH 012/165] fix: add electricity param for add_existing_baseyear rule in perfect foresight workflow --- rules/solve_perfect.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 8fb27f6ef2..3660dbf152 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -5,6 +5,7 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), + electricity=config_provider("electricity"), existing_capacities=config_provider("existing_capacities"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), From 79526a78cead02fa9e0d263d0b12a12b7e5e6fa3 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 13:02:51 +0200 Subject: [PATCH 013/165] feat: improve workflow implementation for more efficient execution of clean_pecd_data with handling of not available pecd years --- rules/build_electricity.smk | 12 ++++++------ scripts/build_renewable_profiles_pecd.py | 7 ++++++- scripts/build_tyndp_h2_network.py | 3 ++- scripts/clean_pecd_data.py | 7 ------- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 614cb0e112..ff056735b6 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -383,12 +383,12 @@ rule build_renewable_profiles: def input_data_pecd(w): return { - f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + pyear + ".csv") - for pyear in [ - "2030", - "2040", - "2050", - ] # complete PECD data is available for the years 2030, 2040 + f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv") + for pyear in set( + config_provider("scenario", "planning_horizons")(w) + ).intersection([2030, 2040]) + # Complete PECD data is only available for the years 2030, 2040 + # TODO: adjust if udpated 2050 data available } diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 3d28633ae8..b1dc15e62e 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -58,9 +58,14 @@ if int(year) not in [2030, 2040, 2050]: year = np.clip(10 * (year // 10), 2030, 2050) logger.warning( - "Planning horizon doesn't match available TYNDP data. " + "Planning horizon doesn't match available TYNDP PECD data. " f"Falling back to previous available year {year}." ) + if year == 2050: + logger.warning( + "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." + ) + year = 2040 profile = ( pd.read_csv( diff --git a/scripts/build_tyndp_h2_network.py b/scripts/build_tyndp_h2_network.py index 4aa7ec789d..86c5a2bf0d 100644 --- a/scripts/build_tyndp_h2_network.py +++ b/scripts/build_tyndp_h2_network.py @@ -10,6 +10,7 @@ import logging +import numpy as np import pandas as pd from _helpers import ( configure_logging, @@ -60,7 +61,7 @@ def load_h2_interzonal_connections(fn, scenario="GA", pyear=2030): "Planning horizon doesn't match available TYNDP data. " "Falling back to closest available year between 2030 and 2050." ) - pyear = min(max(2030, 5 * round(pyear / 5)), 2050) + pyear = np.clip(5 * (pyear // 5), 2030, 2050) scenario_dict = { "GA": "Global Ambition", "DE": "Distributed Energy", diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index a14c5c3b45..89bbf78248 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -104,13 +104,6 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: ) fn_pecd = snakemake.input.fn_pecd - if pyear == "2050": - logger.warning( - "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." - ) - pyear = "2040" - fn_pecd = fn_pecd.replace("2050", "2040") - # Load and prep electricity demand tqdm_kwargs = { "ascii": False, From ba5458da68fd24ada97ccb7000a0ba27a5b5dcf9 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 14:09:41 +0200 Subject: [PATCH 014/165] fix: hotfix for add_brownfield p_max_pu until tyndp generators are added. Needs to be reverted later --- rules/solve_myopic.smk | 1 + scripts/add_brownfield.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 7b6eb06a15..692785fa7c 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -83,6 +83,7 @@ rule add_brownfield: ), threshold_capacity=config_provider("existing_capacities", "threshold_capacity"), snapshots=config_provider("snapshots"), + electricity=config_provider("electricity"), drop_leap_day=config_provider("enable", "drop_leap_day"), carriers=config_provider("electricity", "renewable_carriers"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1e92d6558c..96bb1a4e9b 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -218,7 +218,10 @@ def adjust_renewable_profiles(n, input_profiles, params, year): pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill() ) - for carrier in params["carriers"]: + # TODO: hotfix remove filter for tyndp_renewable_profiles after tyndp generators are added + for carrier in set(params["carriers"]) - set( + params["electricity"]["tyndp_renewable_profiles"]["technologies"] + ): if carrier == "hydro": continue From 81402fe4bd47827f5bbb95ee38132115f8d1316e Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 14:13:26 +0200 Subject: [PATCH 015/165] doc: add description of new configuration option to configtables --- doc/configtables/electricity.csv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 4ea09ef663..0a738bc50e 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -30,6 +30,9 @@ everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignit conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound." ,,, renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. +tyndp_renewable_profiles,,, +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. +-- technologies,--,list,Select the renewable technologies for which PECD renewable profiles are used. Carriers not listed use default renewable profiles. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. From c1fbe1392b26cefe4bb9ee1099c648617f06ceed Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 14:17:50 +0200 Subject: [PATCH 016/165] doc: add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index df7a923699..15e69b7671 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,6 +13,8 @@ Release Notes **Changes** +* Add processing and preparation of TYNDP 2024 PECD renewable profiles instead of default renewable profiles using atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. + * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards. * Add the TYNDP electricity demand as an exogenously set demand (https://github.com/open-energy-transition/open-tyndp/pull/14). This requires the default PyPSA-Eur modelling to be explicitly disabled. The TYNDP electricity demand depends on the planning year, necessitating a different approach to the default PyPSA-Eur one. Wildcards are introduced and load is attached in `prepare_sector_network`. From b684567d5bb8e43939ce6be07edb28e38899fcf8 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 16:01:29 +0200 Subject: [PATCH 017/165] feat: add retrieval rule for PECD data from google drive until data on Zenodo --- rules/build_electricity.smk | 5 ++- rules/retrieve.smk | 14 ++++++++ scripts/clean_pecd_data.py | 2 +- scripts/retrieve_tyndp_pecd_data.py | 56 +++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 scripts/retrieve_tyndp_pecd_data.py diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index ff056735b6..e3e596bdb6 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -435,7 +435,7 @@ rule clean_pecd_data: input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), - fn_pecd="/Users/daniel/Desktop/Work/OET/Projects/open-tyndp/data/2024/20250313_ENTSO-E_ENTSOG_TYNDP_2024_Scenarios_Inputs/PECD/{planning_horizons}/", + fn_pecd="data/tyndp_2024_bundle/PECD", output: pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), log: @@ -451,6 +451,9 @@ rule clean_pecd_data: "../scripts/clean_pecd_data.py" +ruleorder: retrieve_tyndp_pecd_data > clean_pecd_data + + rule build_monthly_prices: input: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 4f2b4646de..a50c261347 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -185,6 +185,20 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" script: "../scripts/retrieve_tyndp_bundle.py" + rule retrieve_tyndp_pecd_data: + params: + tyndp_bundle="data/tyndp_2024_bundle", + # TODO Integrate into Zenodo tyndp data bundle + output: + dir=directory("data/tyndp_2024_bundle/PECD"), + log: + "logs/retrieve_tyndp_pecd_data.log", + retries: 2 + script: + "../scripts/retrieve_tyndp_pecd_data.py" + + ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data + rule retrieve_countries_centroids: output: "data/countries_centroids.geojson", diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 89bbf78248..78f126b8df 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -37,7 +37,7 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: str): - fn = Path(fn_pecd, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + fn = Path(fn_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") if not os.path.isfile(fn): return None diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py new file mode 100644 index 0000000000..3a780113ca --- /dev/null +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: 2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +The TYNDP PECD data contains input data for the 2024 TYNDP scenario building process. + +This rule downloads the TYNDP PECD data from `a gdrive and extracts it in the ``data/tyndp_2024_bundle`` +subdirectory, such that all files of the TYNDP bundle are stored in it. + +**Outputs** + +- ``data/tyndp_2024_bundle/PECD``: PECD input data for TYNDP 2024 scenario building + +""" + +import logging +import os +import zipfile + +from _helpers import configure_logging, progress_retrieve, set_scenario_config + +logger = logging.getLogger(__name__) + +# Define the base URL +url = "https://drive.usercontent.google.com/download?id=1kgCQON-XXxRgDL7gX_uMulCV25RCFr4F&export=download&authuser=0&confirm=t" + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_tyndp_bundle") + rootpath = ".." + else: + rootpath = "." + + configure_logging(snakemake) + set_scenario_config(snakemake) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + + to_fn = snakemake.output.dir + tyndp_bundle_fn = snakemake.params["tyndp_bundle"] + to_fn_zp = to_fn + ".zip" + + # download .zip file + logger.info(f"Downloading TYNDP PECD data from '{url}'.") + progress_retrieve(url, to_fn_zp, disable=disable_progress) + + # extract + logger.info("Extracting TYNDP PECD data.") + with zipfile.ZipFile(to_fn_zp, "r") as zip_ref: + zip_ref.extractall(tyndp_bundle_fn) + + # remove .zip file + os.remove(to_fn_zp) + + logger.info(f"TYNDP PECD data available in '{to_fn}'.") From c33f5a332bb86fdd4924cd3dda0df41259e9a152 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 16:17:08 +0200 Subject: [PATCH 018/165] fix: fix ruleorder statement --- rules/build_electricity.smk | 3 --- rules/retrieve.smk | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index e3e596bdb6..4b7886e253 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -451,9 +451,6 @@ rule clean_pecd_data: "../scripts/clean_pecd_data.py" -ruleorder: retrieve_tyndp_pecd_data > clean_pecd_data - - rule build_monthly_prices: input: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index a50c261347..724b248974 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -197,7 +197,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" script: "../scripts/retrieve_tyndp_pecd_data.py" - ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data + ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data rule retrieve_countries_centroids: output: From 16f9c160347e185591b756e04e722e8ccaeb8d1e Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 28 May 2025 17:33:15 +0200 Subject: [PATCH 019/165] fix: remove unneeded inputs to build_renewable_profiles_pecd and update download link for pecd data --- rules/build_electricity.smk | 11 ----------- scripts/retrieve_tyndp_pecd_data.py | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 4b7886e253..8c2bba23cd 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -400,17 +400,6 @@ rule build_renewable_profiles_pecd: planning_horizons=config_provider("scenario", "planning_horizons"), input: unpack(input_data_pecd), - availability_matrix=resources("availability_matrix_{clusters}_{technology}.nc"), - offshore_shapes=resources("offshore_shapes.geojson"), - distance_regions=resources("regions_onshore_base_s_{clusters}.geojson"), - resource_regions=lambda w: ( - resources("regions_onshore_base_s_{clusters}.geojson") - if w.technology in ("onwind", "solar", "solar-hsat") - else resources("regions_offshore_base_s_{clusters}.geojson") - ), - cutout=lambda w: input_cutout( - w, config_provider("renewable", w.technology, "cutout")(w) - ), output: profile=resources("profile_pecd_{clusters}_{technology}.nc"), log: diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 3a780113ca..528111ab2e 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) # Define the base URL -url = "https://drive.usercontent.google.com/download?id=1kgCQON-XXxRgDL7gX_uMulCV25RCFr4F&export=download&authuser=0&confirm=t" +url = "https://drive.usercontent.google.com/download?id=15YF5_DIhIKrkJvhwbzbj3FKTyTzrtfda&export=download&authuser=0&confirm=t" if __name__ == "__main__": if "snakemake" not in globals(): From df8e1946474627e1b96354ba8c355150bf420550 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 28 May 2025 17:33:48 +0200 Subject: [PATCH 020/165] fix: set file outputs for retrieve_tyndp_bundle --- rules/retrieve.smk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index c325020b04..7ef8295cac 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -179,8 +179,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" h2_reference_grid="data/tyndp_2024_bundle/Line data/ReferenceGrid_Hydrogen.xlsx", electricity_demand=directory("data/tyndp_2024_bundle/Demand Profiles"), h2_imports="data/tyndp_2024_bundle/Hydrogen/H2 IMPORTS GENERATORS PROPERTIES.xlsx", - offshore_nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), - offshore_grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), + offshore_nodes="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", + offshore_grid="data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx", log: "logs/retrieve_tyndp_bundle.log", retries: 2 From 361e8014ed9180331531dff34bed0e69dcec6f3e Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 28 May 2025 17:44:59 +0200 Subject: [PATCH 021/165] feat: assume non extendable links for missing data --- scripts/clean_tyndp_offshore_hubs.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/clean_tyndp_offshore_hubs.py b/scripts/clean_tyndp_offshore_hubs.py index 0e25024684..f2c3a60175 100644 --- a/scripts/clean_tyndp_offshore_hubs.py +++ b/scripts/clean_tyndp_offshore_hubs.py @@ -145,6 +145,13 @@ def load_offshore_grid( grid_costs, how="left", on=["bus0", "bus1", "pyear", "scenario", "carrier"] ) + # Assume non-extendable when missing data + # TODO Validate assumption + grid["p_nom_extendable"] = ~grid.isna().any(axis=1) + grid[["capital_cost", "marginal_cost"]] = grid[ + ["capital_cost", "marginal_cost"] + ].fillna(0) + return grid From f2d5aa34d7236ac08404907aa5e8eb87b71ed72f Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 2 Jun 2025 13:57:33 +0200 Subject: [PATCH 022/165] fix: integrate new config for rule --- config/config.default.yaml | 1 - rules/build_sector.smk | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 9c38bbbca8..b1dcfbc1e6 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -799,7 +799,6 @@ sector: oil: 125 offshore_hubs: false - # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry industry: St_primary_fraction: diff --git a/rules/build_sector.smk b/rules/build_sector.smk index e0dadf4401..345b7070c6 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1338,6 +1338,9 @@ if config["sector"]["h2_topology_tyndp"]: script: "../scripts/build_tyndp_h2_imports.py" + +if config["sector"]["offshore_hubs"]: + rule clean_tyndp_offshore_hubs: params: planning_horizons=config_provider("scenario", "planning_horizons"), From 5328939b4a7402eacf084c9366a36733382817fc Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 2 Jun 2025 14:22:59 +0200 Subject: [PATCH 023/165] refactor: rename clean_tyndp_offshore_hubs to build_tyndp_offshore_hubs to match conventions --- rules/build_sector.smk | 15 ++++++++++++--- ...shore_hubs.py => build_tyndp_offshore_hubs.py} | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) rename scripts/{clean_tyndp_offshore_hubs.py => build_tyndp_offshore_hubs.py} (97%) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 345b7070c6..e85faeb8cc 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1341,7 +1341,7 @@ if config["sector"]["h2_topology_tyndp"]: if config["sector"]["offshore_hubs"]: - rule clean_tyndp_offshore_hubs: + rule build_tyndp_offshore_hubs: params: planning_horizons=config_provider("scenario", "planning_horizons"), scenario=config_provider("tyndp_scenario"), @@ -1349,10 +1349,19 @@ if config["sector"]["offshore_hubs"]: nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), output: - buses=resources("offshore_buses.csv"), + offshore_buses=resources("offshore_buses.csv"), offshore_grid=resources("offshore_grid.csv"), + log: + logs("build_tyndp_offshore_hubs.log"), + benchmark: + benchmarks("build_tyndp_offshore_hubs") + threads: 1 + resources: + mem_mb=4000, + conda: + "../envs/environment.yaml" script: - "../scripts/clean_tyndp_offshore_hubs.py" + "../scripts/build_tyndp_offshore_hubs.py" rule prepare_sector_network: diff --git a/scripts/clean_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py similarity index 97% rename from scripts/clean_tyndp_offshore_hubs.py rename to scripts/build_tyndp_offshore_hubs.py index f2c3a60175..abf012bfae 100644 --- a/scripts/clean_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -160,7 +160,7 @@ def load_offshore_grid( from _helpers import mock_snakemake snakemake = mock_snakemake( - "clean_tyndp_offshore_hubs", configfiles="config/test/config.tyndp.yaml" + "build_tyndp_offshore_hubs", configfiles="config/test/config.tyndp.yaml" ) configure_logging(snakemake) @@ -177,5 +177,5 @@ def load_offshore_grid( ) # Save data - nodes.to_csv(snakemake.output.buses, index=False) + nodes.to_csv(snakemake.output.offshore_buses, index=False) grid.to_csv(snakemake.output.offshore_grid, index=False) From d5bc7eae6c6e21e0fe8de90cdf7632f60cc79259 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 2 Jun 2025 15:43:35 +0200 Subject: [PATCH 024/165] doc: update license identifier --- scripts/add_brownfield.py | 2 +- scripts/build_renewable_profiles_pecd.py | 2 +- scripts/clean_pecd_data.py | 2 +- scripts/retrieve_tyndp_pecd_data.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 96bb1a4e9b..cf4eb621bf 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH and contributors to PyPSA-Eur # # SPDX-License-Identifier: MIT """ diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index b1dc15e62e..23cee8cd9a 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH # # SPDX-License-Identifier: MIT """ diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 78f126b8df..a41474bd8d 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH # # SPDX-License-Identifier: MIT """ diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 528111ab2e..47d87a9539 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2024 The PyPSA-Eur Authors +# SPDX-FileCopyrightText: Open Energy Transition gGmbH # # SPDX-License-Identifier: MIT """ From 500082ccd414e4a1e31ceb301c6f133719709b20 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 3 Jun 2025 12:17:07 +0200 Subject: [PATCH 025/165] feat: update pecd retrieval to use gcp storage --- scripts/retrieve_tyndp_pecd_data.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 47d87a9539..a24bbc3675 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -16,19 +16,20 @@ import logging import os import zipfile +from pathlib import Path from _helpers import configure_logging, progress_retrieve, set_scenario_config logger = logging.getLogger(__name__) # Define the base URL -url = "https://drive.usercontent.google.com/download?id=15YF5_DIhIKrkJvhwbzbj3FKTyTzrtfda&export=download&authuser=0&confirm=t" +url = "https://storage.googleapis.com/open-tyndp-data-store/PECD.zip" if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("retrieve_tyndp_bundle") + snakemake = mock_snakemake("retrieve_tyndp_pecd_data") rootpath = ".." else: rootpath = "." @@ -38,7 +39,7 @@ disable_progress = snakemake.config["run"].get("disable_progressbar", False) to_fn = snakemake.output.dir - tyndp_bundle_fn = snakemake.params["tyndp_bundle"] + tyndp_bundle_fn = Path(rootpath, snakemake.params["tyndp_bundle"]) to_fn_zp = to_fn + ".zip" # download .zip file From 1432a36f4ad30fc95ed698c37836a89c0c73ac26 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 3 Jun 2025 17:17:00 +0200 Subject: [PATCH 026/165] feat: WIP introduce offshore reference grid in the network (assuming absolute investment cost) --- rules/build_sector.smk | 11 ++ scripts/_helpers.py | 7 +- scripts/build_tyndp_offshore_hubs.py | 17 +-- scripts/prepare_sector_network.py | 174 ++++++++++++++++++++++++++- 4 files changed, 195 insertions(+), 14 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index e85faeb8cc..c0f5c048a9 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1400,6 +1400,7 @@ rule prepare_sector_network: ), load_source=config_provider("load", "source"), scaling_factor=config_provider("load", "scaling_factor"), + offshore_hubs=config_provider("sector", "offshore_hubs"), input: unpack(input_profile_offwind), unpack(input_heat_source_power), @@ -1548,6 +1549,16 @@ rule prepare_sector_network: if config_provider("sector", "h2_topology_tyndp")(w) else [] ), + offshore_buses=lambda w: ( + resources("offshore_buses.csv") + if config_provider("sector", "offshore_hubs")(w) + else [] + ), + offshore_grid=lambda w: ( + resources("offshore_grid.csv") + if config_provider("sector", "offshore_hubs")(w) + else [] + ), output: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 8e1142c1c6..c1522c19b3 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1089,8 +1089,11 @@ def load_cutout( return cutout -def make_index(c, carrier): - return carrier + " " + c.bus0 + " -> " + c.bus1 +def make_index(c, carrier="", connector="->"): + idx = [c.bus0, connector, c.bus1] + if carrier: + idx = [carrier] + idx + return " ".join(idx) def extract_grid_data_tyndp( diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index abf012bfae..783aed329d 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -97,8 +97,8 @@ def load_offshore_grid( "SCENARIO": "scenario", "MARKET": "carrier", "CAPACITY": "p_nom", - "CAPEX": "capital_cost", - "OPEX": "marginal_cost", + "CAPEX": "investment", + "OPEX": "opex", } scenario_dict = { @@ -133,12 +133,17 @@ def load_offshore_grid( .rename(columns=column_dict) .query("pyear in @planning_horizons") ) + grid_costs[["investment", "opex"]] = grid_costs[["investment", "opex"]].mul( + 1e6 + ) # MEUR to EUR grid_costs["carrier"] = grid_costs["carrier"].replace("E", "DC") grid_costs["scenario"] = grid_costs["scenario"].replace(scenario_dict) # Rename UK in GB - grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB") - grid_costs[["bus0", "bus1"]] = grid_costs[["bus0", "bus1"]].replace("UK", "GB") + grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB", regex=True) + grid_costs[["bus0", "bus1"]] = grid_costs[["bus0", "bus1"]].replace( + "UK", "GB", regex=True + ) # Merge information grid = grid.merge( @@ -148,9 +153,7 @@ def load_offshore_grid( # Assume non-extendable when missing data # TODO Validate assumption grid["p_nom_extendable"] = ~grid.isna().any(axis=1) - grid[["capital_cost", "marginal_cost"]] = grid[ - ["capital_cost", "marginal_cost"] - ].fillna(0) + grid[["investment", "opex"]] = grid[["investment", "opex"]].fillna(0) return grid diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1b07de7bc2..ad3ab7312d 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -25,6 +25,7 @@ from scripts._helpers import ( configure_logging, get, + make_index, set_scenario_config, update_config_from_wildcards, ) @@ -51,7 +52,7 @@ logger = logging.getLogger(__name__) -def define_spatial(nodes, options, buses_h2_file=None): +def define_spatial(nodes, options, offshore_buses_fn=None, buses_h2_file=None): """ Namespace for spatial. @@ -70,11 +71,32 @@ def define_spatial(nodes, options, buses_h2_file=None): - regional_oil_demand : bool - regional_coal_demand : bool buses_h2_file : str - Path to CSV file containing TYNDP H2 buses information + Path to the file containing TYNDP H2 buses information. + offshore_buses_fn : str + Path to the file containing offshore bus data. """ spatial.nodes = nodes + # offshore hubs + + if options.get("offshore_hubs") and offshore_buses_fn: + spatial.offshore_hubs = SimpleNamespace() + offshore_buses = pd.read_csv(offshore_buses_fn, index_col=0) + offshore_buses_h2 = offshore_buses.set_index(offshore_buses.index + " H2") + spatial.offshore_hubs.nodes = offshore_buses.index + spatial.offshore_hubs.nodes_h2 = offshore_buses_h2.index + spatial.offshore_hubs.x = offshore_buses.x + spatial.offshore_hubs.y = offshore_buses.y + spatial.offshore_hubs.x_h2 = offshore_buses_h2.x + spatial.offshore_hubs.y_h2 = offshore_buses_h2.y + spatial.offshore_hubs.locations = offshore_buses.location + spatial.offshore_hubs.locations_h2 = offshore_buses_h2.location + " H2" + spatial.offshore_hubs.country = offshore_buses.location.str[:2] + spatial.offshore_hubs.country_h2 = offshore_buses_h2.location.str[:2] + spatial.offshore_hubs.type = offshore_buses.type + spatial.offshore_hubs.type_h2 = offshore_buses_h2.type + # biomass spatial.biomass = SimpleNamespace() @@ -3001,6 +3023,132 @@ def add_gas_and_h2_infrastructure( add_h2_pipeline_new(n=n, costs=costs, logger=logger) +def add_offshore_hubs( + n: pypsa.Network, + pyear: int, + offshore_grid_fn: str, + costs: pd.DataFrame, + spatial: spatial, + logger: logging.Logger, + nyears: float = 1, +): + """ + Add offshore hubs to the network model. + + Parameters + ---------- + n : pypsa.Network + The network object to add offshore hubs to. + pyear: int + Planning horizon used to filter which reference grid data to include. + offshore_grid_fn : str + Path to the file containing offshore grid configuration data. + costs : pd.DataFrame + Technology costs assumptions. + spatial : object, optional + Object containing spatial information about nodes and their locations. + logger : logging.Logger, optional + Logger for output messages. If None, no logging is performed. + nyears : float + Number of years for which to scale the investment costs. + + Returns + ------- + None + Modifies the network object in-place by adding offshore hubs + + Notes + ----- + Components added to the network: + - Offshore DC and H2 buses + - Offshore DC and H2 grid + """ + logger.info("Adding offshore hubs") + + n.add( + "Bus", + spatial.offshore_hubs.nodes, + x=spatial.offshore_hubs.x, + y=spatial.offshore_hubs.y, + location=spatial.offshore_hubs.locations, + country=spatial.offshore_hubs.country, + type=spatial.offshore_hubs.type, + carrier="AC", + unit="MWh_el", + v_nom=380, + ) + + n.add( + "Bus", + spatial.offshore_hubs.nodes_h2, + x=spatial.offshore_hubs.x_h2, + y=spatial.offshore_hubs.y_h2, + location=spatial.offshore_hubs.locations_h2, + country=spatial.offshore_hubs.country_h2, + type=spatial.offshore_hubs.type_h2, + carrier="H2", + unit="MWh_LHV", + ) + + offshore_grid = pd.read_csv(offshore_grid_fn).query("pyear==@pyear") + offshore_grid["length"] = offshore_grid.apply(haversine, axis=1, args=(n,)) + annuity_factor = calculate_annuity(costs["lifetime"], costs["discount rate"]) + + offshore_grid_dc = offshore_grid.query("carrier=='DC'").copy() + offshore_grid_dc.index = offshore_grid_dc.apply( + lambda x: f"{x.bus0}-{x.bus1}-DC", axis=1 + ) + offshore_grid_dc.loc[:, "capital_cost"] = ( + annuity_factor.get("HVDC submarine") + * offshore_grid_dc["investment"] + * nyears + / offshore_grid_dc["p_nom"] + + offshore_grid_dc["opex"] * nyears / offshore_grid_dc["p_nom"] + ) + n.add( + "Link", + offshore_grid_dc.index, + bus0=offshore_grid_dc.bus0, + bus1=offshore_grid_dc.bus1, + p_nom_extendable=offshore_grid_dc.p_nom_extendable, + p_nom=offshore_grid_dc.p_nom, + length=offshore_grid_dc.length, + p_min_pu=offshore_grid_dc.p_min_pu, + p_max_pu=offshore_grid_dc.p_max_pu, + capital_cost=offshore_grid_dc.capital_cost, # TODO Validate units + carrier="DC", + lifetime=costs.at["HVDC submarine", "lifetime"], + ) + + offshore_grid_h2 = offshore_grid.query("carrier=='H2'").copy() + offshore_grid_h2.index = offshore_grid_h2.apply( + make_index, axis=1, args=("H2 pipeline",) + ) + offshore_grid_h2.loc[:, "capital_cost"] = ( + annuity_factor.get("H2 (g) submarine pipeline") + * offshore_grid_h2["investment"] + * nyears + / offshore_grid_h2["p_nom"] + + offshore_grid_h2["opex"] * nyears / offshore_grid_h2["p_nom"] + ) + n.add( + "Link", + offshore_grid_h2.index, + bus0=offshore_grid_h2.bus0, + bus1=offshore_grid_h2.bus1, + p_nom_extendable=offshore_grid_h2.p_nom_extendable, + p_nom=offshore_grid_h2.p_nom, + length=offshore_grid_h2.length, + p_min_pu=offshore_grid_h2.p_min_pu, + p_max_pu=offshore_grid_h2.p_max_pu, + capital_cost=offshore_grid_h2.capital_cost, # TODO Validate units + carrier="H2 pipeline", + lifetime=costs.at["H2 (g) submarine pipeline", "lifetime"], + ) + + print("ok") + + def check_land_transport_shares(shares): # Sums up the shares, ignoring None values total_share = sum(filter(None, shares)) @@ -7068,9 +7216,9 @@ def add_import_options( snakemake = mock_snakemake( "prepare_sector_network", opts="", - clusters="10", + clusters="all", sector_opts="", - planning_horizons="2050", + planning_horizons="2030", ) configure_logging(snakemake) # pylint: disable=E0606 @@ -7145,7 +7293,12 @@ def add_import_options( heating_efficiencies = pd.read_csv(fn, index_col=[1, 0]).loc[year] buses_h2_file = snakemake.input.buses_h2 if options["h2_topology_tyndp"] else None - spatial = define_spatial(pop_layout.index, options, buses_h2_file=buses_h2_file) + spatial = define_spatial( + pop_layout.index, + options, + offshore_buses_fn=snakemake.input.offshore_buses, + buses_h2_file=buses_h2_file, + ) if snakemake.params.foresight in ["overnight", "myopic", "perfect"]: add_lifetime_wind_solar(n, costs) @@ -7198,6 +7351,17 @@ def add_import_options( logger=logger, ) + if snakemake.params.offshore_hubs: + add_offshore_hubs( + n=n, + pyear=int(snakemake.wildcards.planning_horizons), + offshore_grid_fn=snakemake.input.offshore_grid, + costs=costs, + spatial=spatial, + logger=logger, + nyears=nyears, + ) + add_battery_stores(n=n, nodes=pop_layout.index, costs=costs) if options["transport"]: From 40b9f46e1b9d6dda01d740c35f9dab90cd537d24 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 4 Jun 2025 10:53:56 +0200 Subject: [PATCH 027/165] feat: switch to relative costs (EUR/kW and EUR/kW/a) and define a specific function --- scripts/build_tyndp_offshore_hubs.py | 10 +- scripts/prepare_sector_network.py | 156 +++++++++++++++++---------- 2 files changed, 103 insertions(+), 63 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 783aed329d..06bf759e14 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -97,7 +97,7 @@ def load_offshore_grid( "SCENARIO": "scenario", "MARKET": "carrier", "CAPACITY": "p_nom", - "CAPEX": "investment", + "CAPEX": "capex", "OPEX": "opex", } @@ -133,9 +133,9 @@ def load_offshore_grid( .rename(columns=column_dict) .query("pyear in @planning_horizons") ) - grid_costs[["investment", "opex"]] = grid_costs[["investment", "opex"]].mul( - 1e6 - ) # MEUR to EUR + grid_costs[["capex", "opex"]] = grid_costs[["capex", "opex"]].mul( + 1e3 + ) # EUR/kW to EUR/MW grid_costs["carrier"] = grid_costs["carrier"].replace("E", "DC") grid_costs["scenario"] = grid_costs["scenario"].replace(scenario_dict) @@ -153,7 +153,7 @@ def load_offshore_grid( # Assume non-extendable when missing data # TODO Validate assumption grid["p_nom_extendable"] = ~grid.isna().any(axis=1) - grid[["investment", "opex"]] = grid[["investment", "opex"]].fillna(0) + grid[["capex", "opex"]] = grid[["capex", "opex"]].fillna(0) return grid diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index ad3ab7312d..1f654d75fa 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3023,88 +3023,59 @@ def add_gas_and_h2_infrastructure( add_h2_pipeline_new(n=n, costs=costs, logger=logger) -def add_offshore_hubs( +def add_offshore_grid( n: pypsa.Network, pyear: int, offshore_grid_fn: str, costs: pd.DataFrame, - spatial: spatial, logger: logging.Logger, nyears: float = 1, ): """ - Add offshore hubs to the network model. + Add offshore grid connections to the network model. + + This function reads offshore grid configuration data and adds both DC and H2 pipeline links to the network. Parameters ---------- n : pypsa.Network - The network object to add offshore hubs to. - pyear: int + The network object to add offshore grid connections to. + pyear : int Planning horizon used to filter which reference grid data to include. offshore_grid_fn : str Path to the file containing offshore grid configuration data. costs : pd.DataFrame Technology costs assumptions. - spatial : object, optional - Object containing spatial information about nodes and their locations. - logger : logging.Logger, optional + logger : logging.Logger Logger for output messages. If None, no logging is performed. - nyears : float + nyears : float, default 1 Number of years for which to scale the investment costs. Returns ------- None - Modifies the network object in-place by adding offshore hubs + Modifies the network object in-place by adding offshore grid links. - Notes - ----- - Components added to the network: - - Offshore DC and H2 buses - - Offshore DC and H2 grid - """ - logger.info("Adding offshore hubs") - n.add( - "Bus", - spatial.offshore_hubs.nodes, - x=spatial.offshore_hubs.x, - y=spatial.offshore_hubs.y, - location=spatial.offshore_hubs.locations, - country=spatial.offshore_hubs.country, - type=spatial.offshore_hubs.type, - carrier="AC", - unit="MWh_el", - v_nom=380, - ) + The capital costs are calculated as: + (annuity_factor * capex + opex) * nyears - n.add( - "Bus", - spatial.offshore_hubs.nodes_h2, - x=spatial.offshore_hubs.x_h2, - y=spatial.offshore_hubs.y_h2, - location=spatial.offshore_hubs.locations_h2, - country=spatial.offshore_hubs.country_h2, - type=spatial.offshore_hubs.type_h2, - carrier="H2", - unit="MWh_LHV", - ) + """ + logger.info("Adding offshore grid connections") offshore_grid = pd.read_csv(offshore_grid_fn).query("pyear==@pyear") - offshore_grid["length"] = offshore_grid.apply(haversine, axis=1, args=(n,)) annuity_factor = calculate_annuity(costs["lifetime"], costs["discount rate"]) + # Add DC grid connections offshore_grid_dc = offshore_grid.query("carrier=='DC'").copy() offshore_grid_dc.index = offshore_grid_dc.apply( lambda x: f"{x.bus0}-{x.bus1}-DC", axis=1 ) offshore_grid_dc.loc[:, "capital_cost"] = ( - annuity_factor.get("HVDC submarine") - * offshore_grid_dc["investment"] - * nyears - / offshore_grid_dc["p_nom"] - + offshore_grid_dc["opex"] * nyears / offshore_grid_dc["p_nom"] - ) + annuity_factor.get("HVDC submarine") * offshore_grid_dc["capex"] + + offshore_grid_dc["opex"] + ) * nyears + n.add( "Link", offshore_grid_dc.index, @@ -3112,25 +3083,23 @@ def add_offshore_hubs( bus1=offshore_grid_dc.bus1, p_nom_extendable=offshore_grid_dc.p_nom_extendable, p_nom=offshore_grid_dc.p_nom, - length=offshore_grid_dc.length, p_min_pu=offshore_grid_dc.p_min_pu, p_max_pu=offshore_grid_dc.p_max_pu, - capital_cost=offshore_grid_dc.capital_cost, # TODO Validate units + capital_cost=offshore_grid_dc.capital_cost, carrier="DC", lifetime=costs.at["HVDC submarine", "lifetime"], ) + # Add H2 pipeline connections offshore_grid_h2 = offshore_grid.query("carrier=='H2'").copy() offshore_grid_h2.index = offshore_grid_h2.apply( make_index, axis=1, args=("H2 pipeline",) ) offshore_grid_h2.loc[:, "capital_cost"] = ( - annuity_factor.get("H2 (g) submarine pipeline") - * offshore_grid_h2["investment"] - * nyears - / offshore_grid_h2["p_nom"] - + offshore_grid_h2["opex"] * nyears / offshore_grid_h2["p_nom"] - ) + annuity_factor.get("H2 (g) submarine pipeline") * offshore_grid_h2["capex"] + + offshore_grid_h2["opex"] + ) * nyears + n.add( "Link", offshore_grid_h2.index, @@ -3138,15 +3107,86 @@ def add_offshore_hubs( bus1=offshore_grid_h2.bus1, p_nom_extendable=offshore_grid_h2.p_nom_extendable, p_nom=offshore_grid_h2.p_nom, - length=offshore_grid_h2.length, p_min_pu=offshore_grid_h2.p_min_pu, p_max_pu=offshore_grid_h2.p_max_pu, - capital_cost=offshore_grid_h2.capital_cost, # TODO Validate units + capital_cost=offshore_grid_h2.capital_cost, carrier="H2 pipeline", lifetime=costs.at["H2 (g) submarine pipeline", "lifetime"], ) - print("ok") + +def add_offshore_hubs( + n: pypsa.Network, + pyear: int, + offshore_grid_fn: str, + costs: pd.DataFrame, + spatial: spatial, + logger: logging.Logger, + nyears: float = 1, +): + """ + Add offshore hubs and grid connections to the network model. + + This function creates offshore hub infrastructure by adding both the physical + hubs (buses) and their interconnecting grid (DC and H2 pipeline links). + + Parameters + ---------- + n : pypsa.Network + The network object to add offshore hubs and grid to. + pyear: int + Planning horizon used to filter which reference grid data to include. + offshore_grid_fn : str + Path to the file containing offshore grid configuration data. + costs : pd.DataFrame + Technology costs assumptions. + spatial : object, optional + Object containing spatial information about nodes and their locations. + logger : logging.Logger, optional + Logger for output messages. If None, no logging is performed. + nyears : float + Number of years for which to scale the investment costs. + + Returns + ------- + None + Modifies the network object in-place by adding offshore hubs + + Notes + ----- + Components added to the network: + - Offshore DC and H2 buses + - Offshore DC and H2 grid + """ + logger.info("Adding offshore hubs") + + n.add( + "Bus", + spatial.offshore_hubs.nodes, + x=spatial.offshore_hubs.x, + y=spatial.offshore_hubs.y, + location=spatial.offshore_hubs.locations, + country=spatial.offshore_hubs.country, + type=spatial.offshore_hubs.type, + carrier="AC", + unit="MWh_el", + v_nom=380, + ) + + n.add( + "Bus", + spatial.offshore_hubs.nodes_h2, + x=spatial.offshore_hubs.x_h2, + y=spatial.offshore_hubs.y_h2, + location=spatial.offshore_hubs.locations_h2, + country=spatial.offshore_hubs.country_h2, + type=spatial.offshore_hubs.type_h2, + carrier="H2", + unit="MWh_LHV", + ) + + # Add offshore DC and H2 grid connections + add_offshore_grid(n, pyear, offshore_grid_fn, costs, logger, nyears) def check_land_transport_shares(shares): From b140dce284e7158c241dfc665ea1c1c0a539bea7 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 4 Jun 2025 11:49:10 +0200 Subject: [PATCH 028/165] feat: map AC nodes to H2 Z2 nodes for H2 offshore grid --- scripts/build_tyndp_offshore_hubs.py | 4 +++- scripts/prepare_sector_network.py | 28 +++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 06bf759e14..e34741dd08 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -50,7 +50,9 @@ def load_offshore_hubs(fn: str): nodes = gpd.GeoDataFrame(nodes, geometry="geometry", crs=GEO_CRS) # rename UK in GB - nodes[["Bus", "location"]] = nodes[["Bus", "location"]].replace("UK", "GB") + nodes[["Bus", "location"]] = nodes[["Bus", "location"]].replace( + "UK", "GB", regex=True + ) return nodes diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1f654d75fa..9f88dd5706 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -91,7 +91,7 @@ def define_spatial(nodes, options, offshore_buses_fn=None, buses_h2_file=None): spatial.offshore_hubs.x_h2 = offshore_buses_h2.x spatial.offshore_hubs.y_h2 = offshore_buses_h2.y spatial.offshore_hubs.locations = offshore_buses.location - spatial.offshore_hubs.locations_h2 = offshore_buses_h2.location + " H2" + spatial.offshore_hubs.locations_h2 = offshore_buses_h2.location spatial.offshore_hubs.country = offshore_buses.location.str[:2] spatial.offshore_hubs.country_h2 = offshore_buses_h2.location.str[:2] spatial.offshore_hubs.type = offshore_buses.type @@ -3023,6 +3023,23 @@ def add_gas_and_h2_infrastructure( add_h2_pipeline_new(n=n, costs=costs, logger=logger) +def map_h2_buses(n, df): + """ + Map AC buses to H2 Z2 buses. + """ + h2_busmap = ( + n.buses.query( + "~Bus.str.contains('DRES') and carrier=='AC' and type==''" + ).location.str[:2] + + " H2 Z2" + ) + df_mapped = df.assign( + bus0=lambda x: x["bus0"].map(h2_busmap).fillna(x["bus0"]), + bus1=lambda x: x["bus1"].map(h2_busmap).fillna(x["bus1"]), + ) + return df_mapped + + def add_offshore_grid( n: pypsa.Network, pyear: int, @@ -3099,6 +3116,7 @@ def add_offshore_grid( annuity_factor.get("H2 (g) submarine pipeline") * offshore_grid_h2["capex"] + offshore_grid_h2["opex"] ) * nyears + offshore_grid_h2 = map_h2_buses(n, offshore_grid_h2) n.add( "Link", @@ -3115,7 +3133,7 @@ def add_offshore_grid( ) -def add_offshore_hubs( +def add_offshore_hubs_topology( n: pypsa.Network, pyear: int, offshore_grid_fn: str, @@ -3185,6 +3203,10 @@ def add_offshore_hubs( unit="MWh_LHV", ) + # Add power production units + + # Add H2 production units + # Add offshore DC and H2 grid connections add_offshore_grid(n, pyear, offshore_grid_fn, costs, logger, nyears) @@ -7392,7 +7414,7 @@ def add_import_options( ) if snakemake.params.offshore_hubs: - add_offshore_hubs( + add_offshore_hubs_topology( n=n, pyear=int(snakemake.wildcards.planning_horizons), offshore_grid_fn=snakemake.input.offshore_grid, From 77defb1bd2e3f69b87ef7b19c02e0cffc30d5a54 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 4 Jun 2025 11:57:40 +0200 Subject: [PATCH 029/165] doc: add basic release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b1c83afd67..3e7242190f 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,6 +13,8 @@ Release Notes **Changes** +* Introduce offshore wind hubs (https://github.com/open-energy-transition/open-tyndp/pull/54). + * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards. * Add the TYNDP electricity demand as an exogenously set demand (https://github.com/open-energy-transition/open-tyndp/pull/14). This requires the default PyPSA-Eur modelling to be explicitly disabled. The TYNDP electricity demand depends on the planning year, necessitating a different approach to the default PyPSA-Eur one. Wildcards are introduced and load is attached in `prepare_sector_network`. From 1ac7f27762dd6ca3e9e3b49abb73d4f4cc8968bc Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 4 Jun 2025 11:58:34 +0200 Subject: [PATCH 030/165] refactor: rename function to be tyndp specific --- scripts/prepare_sector_network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 9f88dd5706..7bdb04850f 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3040,7 +3040,7 @@ def map_h2_buses(n, df): return df_mapped -def add_offshore_grid( +def add_offshore_grid_tyndp( n: pypsa.Network, pyear: int, offshore_grid_fn: str, @@ -3133,7 +3133,7 @@ def add_offshore_grid( ) -def add_offshore_hubs_topology( +def add_offshore_hubs_tyndp( n: pypsa.Network, pyear: int, offshore_grid_fn: str, @@ -3208,7 +3208,7 @@ def add_offshore_hubs_topology( # Add H2 production units # Add offshore DC and H2 grid connections - add_offshore_grid(n, pyear, offshore_grid_fn, costs, logger, nyears) + add_offshore_grid_tyndp(n, pyear, offshore_grid_fn, costs, logger, nyears) def check_land_transport_shares(shares): @@ -7414,7 +7414,7 @@ def add_import_options( ) if snakemake.params.offshore_hubs: - add_offshore_hubs_topology( + add_offshore_hubs_tyndp( n=n, pyear=int(snakemake.wildcards.planning_horizons), offshore_grid_fn=snakemake.input.offshore_grid, From b474845ec0e384b71c1e308d17c7ea77c8675a31 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 4 Jun 2025 12:10:32 +0200 Subject: [PATCH 031/165] revert: revert change in default config --- scripts/prepare_sector_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 97285ae5ab..ee8957bd58 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -7288,9 +7288,9 @@ def add_import_options( snakemake = mock_snakemake( "prepare_sector_network", opts="", - clusters="all", + clusters="10", sector_opts="", - planning_horizons="2030", + planning_horizons="2050", ) configure_logging(snakemake) # pylint: disable=E0606 From 24bb68427382376b59918b93b10068ecc9dd5c50 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 4 Jun 2025 12:34:25 +0200 Subject: [PATCH 032/165] feat: build electrolysers data --- rules/build_sector.smk | 9 +++++ scripts/build_tyndp_offshore_hubs.py | 56 ++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 2a4446777b..f8db043062 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1358,9 +1358,13 @@ if config["sector"]["offshore_hubs"]: input: nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), + electrolysers=directory( + "data/tyndp_2024_bundle/Offshore hubs/ELECTROLYSER.xlsx" + ), output: offshore_buses=resources("offshore_buses.csv"), offshore_grid=resources("offshore_grid.csv"), + offshore_electrolysers=resources("offshore_electrolysers.csv"), log: logs("build_tyndp_offshore_hubs.log"), benchmark: @@ -1570,6 +1574,11 @@ rule prepare_sector_network: if config_provider("sector", "offshore_hubs")(w) else [] ), + offshore_electrolysers=lambda w: ( + resources("offshore_electrolysers.csv") + if config_provider("sector", "offshore_hubs")(w) + else [] + ), output: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index e34741dd08..82179c2152 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -160,6 +160,57 @@ def load_offshore_grid( return grid +def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[int]): + """ + Load offshore electrolysers data and format data. + + Parameters + ---------- + fn : str + Path to the Excel file containing offshore electrolyser data. + scenario : str + Scenario identifier to filter the grid data. Must be one of the scenario + codes: "DE" (Distributed Energy), "GA" (Global Ambition), or + "NT" (National Trends). + planning_horizons : list[int] + List of planning years to include in the cost data filtering. + + Returns + ------- + pd.DataFrame + DataFrame containing the formatted offshore electrolyser data. + """ + column_dict = { + "NODE": "location", + "OFFSHORE_NODE": "Bus", + "OFFSHORE_NODE_TYPE": "type", + "YEAR": "pyear", + "SCENARIO": "scenario", + "CAPEX": "capex", + "OPEX": "opex", + } + + scenario_dict = { + "Distributed Energy": "DE", + "Global Ambition": "GA", + "National Trends": "NT", + } + + # Load electrolysers data + electrolysers = ( + pd.read_excel( + fn, + sheet_name="COST", + ) + .rename(columns=column_dict) + .query("pyear in @planning_horizons") + .replace({"scenario": scenario_dict}) + .query("scenario == @scenario") + ) + + return electrolysers + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -181,6 +232,11 @@ def load_offshore_grid( snakemake.input.grid, nodes, snakemake.params["scenario"], planning_horizons ) + electrolysers = load_offshore_electrolysers( + snakemake.input.electrolysers, snakemake.params["scenario"], planning_horizons + ) + # Save data nodes.to_csv(snakemake.output.offshore_buses, index=False) grid.to_csv(snakemake.output.offshore_grid, index=False) + electrolysers.to_csv(snakemake.output.offshore_electrolysers, index=False) From c5297b14b13ed1a8c48a944b5cb195c531b34ea4 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 4 Jun 2025 13:03:04 +0200 Subject: [PATCH 033/165] feat: attach offshore electrolysers --- scripts/build_tyndp_offshore_hubs.py | 7 ++- scripts/prepare_sector_network.py | 68 ++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 82179c2152..29ecff256c 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -182,7 +182,7 @@ def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[ """ column_dict = { "NODE": "location", - "OFFSHORE_NODE": "Bus", + "OFFSHORE_NODE": "bus0", "OFFSHORE_NODE_TYPE": "type", "YEAR": "pyear", "SCENARIO": "scenario", @@ -206,8 +206,13 @@ def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[ .query("pyear in @planning_horizons") .replace({"scenario": scenario_dict}) .query("scenario == @scenario") + .assign(bus1=lambda x: x.bus0 + " H2") ) + electrolysers[["capex", "opex"]] = electrolysers[["capex", "opex"]].mul( + 1e3 + ) # EUR/kW to EUR/MW + return electrolysers diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index ee8957bd58..f535ce320b 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3049,6 +3049,67 @@ def map_h2_buses(n, df): return df_mapped +def add_offshore_electrolysers_tyndp( + n: pypsa.Network, + pyear: int, + offshore_electrolysers_fn: str, + costs: pd.DataFrame, + logger: logging.Logger, + nyears: float = 1, +): + """ + Add offshore electrolysers to the network model. + + This function adds offshore electrolysis capacity to the offshore hub buses in the network. + + Parameters + ---------- + n : pypsa.Network + The network object to add offshore generators to. + pyear : int + Planning horizon used to filter which reference generator data to include. + offshore_electrolysers_fn : str + Path to the file containing offshore electrolysers configuration data. + costs : pd.DataFrame + Technology costs assumptions. + logger : logging.Logger + Logger for output messages. If None, no logging is performed. + nyears : float, default 1 + Number of years for which to scale the investment costs. + + Returns + ------- + None + Modifies the network object in-place by adding offshore generators. + """ + logger.info("Adding offshore electrolysers") + + offshore_electrolysers = pd.read_csv(offshore_electrolysers_fn).query( + "pyear==@pyear" + ) + annuity_factor = calculate_annuity(costs["lifetime"], costs["discount rate"]) + offshore_electrolysers.index = ( + offshore_electrolysers.bus0 + " H2 Offshore Electrolysis" + ) + + offshore_electrolysers.loc[:, "capital_cost"] = ( + annuity_factor.get("electrolysis") * offshore_electrolysers["capex"] + + offshore_electrolysers["opex"] + ) * nyears + + n.add( + "Link", + offshore_electrolysers.index, + bus0=offshore_electrolysers.bus0, + bus1=offshore_electrolysers.bus1, + p_nom_extendable=True, + carrier="H2 Electrolysis", + efficiency=costs.at["electrolysis", "efficiency"], + capital_cost=costs.at["electrolysis", "capital_cost"], + lifetime=costs.at["electrolysis", "lifetime"], + ) + + def add_offshore_grid_tyndp( n: pypsa.Network, pyear: int, @@ -3146,6 +3207,7 @@ def add_offshore_hubs_tyndp( n: pypsa.Network, pyear: int, offshore_grid_fn: str, + offshore_electrolysers_fn: str, costs: pd.DataFrame, spatial: spatial, logger: logging.Logger, @@ -3165,6 +3227,8 @@ def add_offshore_hubs_tyndp( Planning horizon used to filter which reference grid data to include. offshore_grid_fn : str Path to the file containing offshore grid configuration data. + offshore_electrolysers_fn : str + Path to the file containing offshore electrolysers configuration data. costs : pd.DataFrame Technology costs assumptions. spatial : object, optional @@ -3215,6 +3279,9 @@ def add_offshore_hubs_tyndp( # Add power production units # Add H2 production units + add_offshore_electrolysers_tyndp( + n, pyear, offshore_electrolysers_fn, costs, logger, nyears + ) # Add offshore DC and H2 grid connections add_offshore_grid_tyndp(n, pyear, offshore_grid_fn, costs, logger, nyears) @@ -7428,6 +7495,7 @@ def add_import_options( n=n, pyear=int(snakemake.wildcards.planning_horizons), offshore_grid_fn=snakemake.input.offshore_grid, + offshore_electrolysers_fn=snakemake.input.offshore_electrolysers, costs=costs, spatial=spatial, logger=logger, From d6903182fa78fa5093306fae481e033f7deccc82 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 4 Jun 2025 13:06:31 +0200 Subject: [PATCH 034/165] refactor: switch functions --- scripts/prepare_sector_network.py | 35 ++++++++++++++++--------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index f535ce320b..425573fade 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3032,23 +3032,6 @@ def add_gas_and_h2_infrastructure( add_h2_pipeline_new(n=n, costs=costs, logger=logger) -def map_h2_buses(n, df): - """ - Map AC buses to H2 Z2 buses. - """ - h2_busmap = ( - n.buses.query( - "~Bus.str.contains('DRES') and carrier=='AC' and type==''" - ).location.str[:2] - + " H2 Z2" - ) - df_mapped = df.assign( - bus0=lambda x: x["bus0"].map(h2_busmap).fillna(x["bus0"]), - bus1=lambda x: x["bus1"].map(h2_busmap).fillna(x["bus1"]), - ) - return df_mapped - - def add_offshore_electrolysers_tyndp( n: pypsa.Network, pyear: int, @@ -3110,6 +3093,24 @@ def add_offshore_electrolysers_tyndp( ) +def map_h2_buses(n, df): + """ + Map AC buses to H2 Z2 buses. + """ + h2_busmap = ( + n.buses.query( + "~Bus.str.contains('DRES') and carrier=='AC' and type==''" + ).location.str[:2] + + " H2 Z2" + ) + df_mapped = df.assign( + bus0=lambda x: x["bus0"].map(h2_busmap).fillna(x["bus0"]), + bus1=lambda x: x["bus1"].map(h2_busmap).fillna(x["bus1"]), + ) + return df_mapped + + + def add_offshore_grid_tyndp( n: pypsa.Network, pyear: int, From d9bc203acbaac910e13079ee4c009fe39a0d40bb Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 4 Jun 2025 16:12:29 +0200 Subject: [PATCH 035/165] feat: build generators data --- rules/build_sector.smk | 7 ++ scripts/build_tyndp_offshore_hubs.py | 117 +++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index f8db043062..ba877ce59a 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1361,10 +1361,12 @@ if config["sector"]["offshore_hubs"]: electrolysers=directory( "data/tyndp_2024_bundle/Offshore hubs/ELECTROLYSER.xlsx" ), + generators=directory("data/tyndp_2024_bundle/Offshore hubs/GENERATOR.xlsx"), output: offshore_buses=resources("offshore_buses.csv"), offshore_grid=resources("offshore_grid.csv"), offshore_electrolysers=resources("offshore_electrolysers.csv"), + offshore_generators=resources("offshore_generators.csv"), log: logs("build_tyndp_offshore_hubs.log"), benchmark: @@ -1579,6 +1581,11 @@ rule prepare_sector_network: if config_provider("sector", "offshore_hubs")(w) else [] ), + offshore_generators=lambda w: ( + resources("offshore_generators.csv") + if config_provider("sector", "offshore_hubs")(w) + else [] + ), output: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 29ecff256c..0d6c205d0f 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -134,18 +134,12 @@ def load_offshore_grid( ) .rename(columns=column_dict) .query("pyear in @planning_horizons") + .replace({"scenario": scenario_dict}) ) grid_costs[["capex", "opex"]] = grid_costs[["capex", "opex"]].mul( 1e3 ) # EUR/kW to EUR/MW grid_costs["carrier"] = grid_costs["carrier"].replace("E", "DC") - grid_costs["scenario"] = grid_costs["scenario"].replace(scenario_dict) - - # Rename UK in GB - grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB", regex=True) - grid_costs[["bus0", "bus1"]] = grid_costs[["bus0", "bus1"]].replace( - "UK", "GB", regex=True - ) # Merge information grid = grid.merge( @@ -154,9 +148,12 @@ def load_offshore_grid( # Assume non-extendable when missing data # TODO Validate assumption - grid["p_nom_extendable"] = ~grid.isna().any(axis=1) + grid["p_nom_extendable"] = ~grid[["capex", "opex"]].isna().any(axis=1) grid[["capex", "opex"]] = grid[["capex", "opex"]].fillna(0) + # Rename UK in GB + grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB", regex=True) + return grid @@ -213,9 +210,108 @@ def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[ 1e3 ) # EUR/kW to EUR/MW + # rename UK in GB + electrolysers[["bus0", "bus1", "location"]] = electrolysers[ + ["bus0", "bus1", "location"] + ].replace("UK", "GB", regex=True) + return electrolysers +def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int]): + """ + Load offshore generators data and format data. + + Parameters + ---------- + fn : str + Path to the Excel file containing offshore generators data. + scenario : str + Scenario identifier to filter the grid data. Must be one of the scenario + codes: "DE" (Distributed Energy), "GA" (Global Ambition), or + "NT" (National Trends). + planning_horizons : list[int] + List of planning years to include in the cost data filtering. + + Returns + ------- + pd.DataFrame + DataFrame containing the formatted offshore generators data. + """ + column_dict = { # TODO + "NODE": "location", + "OFFSHORE_NODE": "bus0", + "OFFSHORE_NODE_TYPE": "type", + "YEAR": "pyear", + "SCENARIO": "scenario", + "TECHNOLOGY": "carrier", + "CAPEX": "capex", + "OPEX": "opex", + "MW": "p_nom_min", + } + + scenario_dict = { + "Distributed Energy": "DE", + "Global Ambition": "GA", + "National Trends": "NT", + } + + # Load generators data + generators = ( + pd.read_excel( + fn, + sheet_name="EXISTING", + ) + .rename(columns=column_dict) + .query("pyear in @planning_horizons") + .replace({"scenario": scenario_dict}) + .query("scenario == @scenario") + .assign( + carrier=lambda x: "offwind-" + + x.carrier.str.lower().replace("_", "-", regex=True) + ) + ) + + # Load costs data + generators_costs = ( + pd.read_excel( + fn, + sheet_name="COST", + ) + .rename(columns=column_dict) + .query("pyear in @planning_horizons") + .replace({"scenario": scenario_dict}) + .query("scenario == @scenario") + .assign( + carrier=lambda x: "offwind-" + + x.carrier.str.lower().replace("_", "-", regex=True) + ) + ) + + generators_costs[["capex", "opex"]] = generators_costs[["capex", "opex"]].mul( + 1e3 + ) # EUR/kW to EUR/MW + + # Merge information + generators = generators_costs.merge( + generators, + how="outer", + on=["bus0", "location", "pyear", "scenario", "type", "carrier"], + ).assign(p_nom_min=lambda x: x["p_nom_min"].fillna(0)) + + # Assume non-extendable when missing data + # TODO Validate assumption + generators["p_nom_extendable"] = ~generators[["capex", "opex"]].isna().any(axis=1) + generators[["capex", "opex"]] = generators[["capex", "opex"]].fillna(0) + + # Rename UK in GB + generators[["bus0", "location"]] = generators[["bus0", "location"]].replace( + "UK", "GB", regex=True + ) + + return generators + + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -241,7 +337,12 @@ def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[ snakemake.input.electrolysers, snakemake.params["scenario"], planning_horizons ) + generators = load_offshore_generators( + snakemake.input.generators, snakemake.params["scenario"], planning_horizons + ) + # Save data nodes.to_csv(snakemake.output.offshore_buses, index=False) grid.to_csv(snakemake.output.offshore_grid, index=False) electrolysers.to_csv(snakemake.output.offshore_electrolysers, index=False) + generators.to_csv(snakemake.output.offshore_generators, index=False) From d3457ce76d2853aeddea16db090cee2352f47e29 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Jun 2025 15:12:57 +0000 Subject: [PATCH 036/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/prepare_sector_network.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 425573fade..69c399b2cc 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3110,7 +3110,6 @@ def map_h2_buses(n, df): return df_mapped - def add_offshore_grid_tyndp( n: pypsa.Network, pyear: int, From 48dcfbaf30f797332aa67fac1997b8b85c38fb82 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 5 Jun 2025 14:32:09 +0200 Subject: [PATCH 037/165] feat: overwrite default lifetime assumptions with tyndp specific values --- config/config.tyndp.yaml | 8 ++++++++ config/test/config.tyndp.yaml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 2acf850f53..64a0d498bc 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -133,6 +133,14 @@ sector: - H2 offshore_hubs: true +costs: + overwrites: + lifetime: + electrolysis: 25 + HVDC submarine: 25 + H2 (g) submarine pipeline: 25 + offwind: 25 + clustering: mode: administrative administrative: diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 519b6417d7..bec2fcc420 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -152,6 +152,14 @@ sector: - H2 offshore_hubs: true +costs: + overwrites: + lifetime: + electrolysis: 25 + HVDC submarine: 25 + H2 (g) submarine pipeline: 25 + offwind: 25 + clustering: mode: administrative administrative: From 2a94652feb8eb712ad9a743dc90c214f7d798022 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 5 Jun 2025 14:34:26 +0200 Subject: [PATCH 038/165] feat: generalize more make_index --- scripts/_helpers.py | 9 ++++----- scripts/clean_tyndp_h2_imports.py | 4 +--- scripts/prepare_sector_network.py | 9 ++------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 6f1ed66c78..6ae89a8def 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1075,10 +1075,9 @@ def load_cutout( return cutout -def make_index(c, carrier="", connector="->"): - idx = [c.bus0, connector, c.bus1] - if carrier: - idx = [carrier] + idx +def make_index(c, cname0="bus0", cname1="bus1", prefix="", connector="->", suffix=""): + idx = [prefix, c[cname0], connector, c[cname1], suffix] + idx = [i for i in idx if i != ""] return " ".join(idx) @@ -1125,6 +1124,6 @@ def extract_grid_data_tyndp( # Combine into unidirectional links and return h2_grid = pd.concat([forward_links, reverse_links]) - h2_grid.index = h2_grid.apply(make_index, axis=1, args=(carrier,)) + h2_grid.index = h2_grid.apply(make_index, axis=1, prefix=carrier) return h2_grid diff --git a/scripts/clean_tyndp_h2_imports.py b/scripts/clean_tyndp_h2_imports.py index 8929cde8d6..2dad03e4d2 100644 --- a/scripts/clean_tyndp_h2_imports.py +++ b/scripts/clean_tyndp_h2_imports.py @@ -116,9 +116,7 @@ def load_import_data(fn, countries_centroids): # Match countries centroids for defining coordinates of import nodes imports = match_centroids(imports, countries_centroids) - imports.index = ( - imports.apply(make_index, axis=1, args=("H2 import",)) + " - " + imports.Band - ) + imports.index = imports.apply(make_index, axis=1, prefix="H2 import") + " - " + imports.Band return imports diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 69c399b2cc..e1de168f82 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -458,11 +458,8 @@ def create_network_topology( candidates_n = candidates[~positive_order].rename(columns=swap_buses) candidates = pd.concat([candidates_p, candidates_n]) - def make_index(c): - return prefix + c.bus0 + connector + c.bus1 - topo = candidates.groupby(["bus0", "bus1"], as_index=False).mean() - topo.index = topo.apply(make_index, axis=1) + topo.index = topo.apply(make_index, axis=1, prefix=prefix, connector=connector) if not bidirectional: topo_reverse = topo.copy() @@ -3179,9 +3176,7 @@ def add_offshore_grid_tyndp( # Add H2 pipeline connections offshore_grid_h2 = offshore_grid.query("carrier=='H2'").copy() - offshore_grid_h2.index = offshore_grid_h2.apply( - make_index, axis=1, args=("H2 pipeline",) - ) + offshore_grid_h2.index = offshore_grid_h2.apply(make_index, axis=1, prefix="H2 pipeline") offshore_grid_h2.loc[:, "capital_cost"] = ( annuity_factor.get("H2 (g) submarine pipeline") * offshore_grid_h2["capex"] + offshore_grid_h2["opex"] From 24d3a634a72e74cfba6d47da03e02977ce6c39f5 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 5 Jun 2025 14:35:12 +0200 Subject: [PATCH 039/165] fix: adjust offshore radial locations --- scripts/build_tyndp_offshore_hubs.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 0d6c205d0f..92b41d9262 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -21,6 +21,8 @@ def load_offshore_hubs(fn: str): """ Load offshore hubs coordinates and format data. + Offshore Hubs (OH) nodes are situated offshore, while Offshore Radial (OR) nodes are located in the homeland market node. + Parameters ---------- fn : str @@ -46,6 +48,9 @@ def load_offshore_hubs(fn: str): sheet_name="NODE", ).rename(columns=column_dict) + mask = nodes["Bus"].str.contains("OH") + nodes.loc[mask, "location"] = nodes.loc[mask, "Bus"] + nodes["geometry"] = nodes.apply(lambda row: Point(row["x"], row["y"]), axis=1) nodes = gpd.GeoDataFrame(nodes, geometry="geometry", crs=GEO_CRS) From b0fd9a487abd249857bfb0194b286513f40d1e8a Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 5 Jun 2025 14:35:56 +0200 Subject: [PATCH 040/165] doc: update units info --- scripts/build_tyndp_offshore_hubs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 92b41d9262..9d131bc532 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -143,7 +143,7 @@ def load_offshore_grid( ) grid_costs[["capex", "opex"]] = grid_costs[["capex", "opex"]].mul( 1e3 - ) # EUR/kW to EUR/MW + ) # kEUR/MW to EUR/MW grid_costs["carrier"] = grid_costs["carrier"].replace("E", "DC") # Merge information @@ -213,7 +213,7 @@ def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[ electrolysers[["capex", "opex"]] = electrolysers[["capex", "opex"]].mul( 1e3 - ) # EUR/kW to EUR/MW + ) # kEUR/MW to EUR/MW # rename UK in GB electrolysers[["bus0", "bus1", "location"]] = electrolysers[ @@ -295,7 +295,7 @@ def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int generators_costs[["capex", "opex"]] = generators_costs[["capex", "opex"]].mul( 1e3 - ) # EUR/kW to EUR/MW + ) # kEUR/MW to EUR/MW # Merge information generators = generators_costs.merge( From 1847f2112d5a7d1a494cd936715fe27f96701f88 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 5 Jun 2025 16:17:50 +0200 Subject: [PATCH 041/165] feat: add offshore generators to the network (without potentials and pecd data) --- scripts/build_tyndp_offshore_hubs.py | 2 +- scripts/prepare_sector_network.py | 80 ++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 9d131bc532..7b5a38954c 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -243,7 +243,7 @@ def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int pd.DataFrame DataFrame containing the formatted offshore generators data. """ - column_dict = { # TODO + column_dict = { "NODE": "location", "OFFSHORE_NODE": "bus0", "OFFSHORE_NODE_TYPE": "type", diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index e1de168f82..795e3d330a 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3029,6 +3029,69 @@ def add_gas_and_h2_infrastructure( add_h2_pipeline_new(n=n, costs=costs, logger=logger) +def add_offshore_generators_tyndp( + n: pypsa.Network, + pyear: int, + offshore_generators_fn: str, + logger: logging.Logger, +): + """ + Add offshore generators to the network model. + + This function adds offshore generation capacity, various offshore wind + turbines (AC and H2), to the offshore hub buses in the network. + + Parameters + ---------- + n : pypsa.Network + The network object to add offshore generators to. + pyear : int + Planning horizon used to filter which reference generator data to include. + offshore_generators : str + Path to the file containing offshore generators configuration data. + logger : logging.Logger + Logger for output messages. If None, no logging is performed. + + Returns + ------- + None + Modifies the network object in-place by adding offshore generators. + """ + logger.info("Adding offshore generators") + + offshore_generators = pd.read_csv(offshore_generators_fn).query("pyear==@pyear") + + mask = offshore_generators["carrier"].str.contains("h2") + offshore_generators.loc[mask, ["bus0", "location"]] = ( + offshore_generators.loc[mask, ["bus0", "location"]] + " H2" + ) + offshore_generators.index = ( + offshore_generators.bus0 + " " + offshore_generators.carrier + ) + + annuity_factor = calculate_annuity(costs["lifetime"], costs["discount rate"]) + offshore_generators.loc[:, "capital_cost"] = ( + annuity_factor.get("electrolysis") * offshore_generators["capex"] + + offshore_generators["opex"] + ) * nyears + + n.add( + "Generator", + offshore_generators.index, + bus=offshore_generators.location, + carrier=offshore_generators.carrier, + p_nom=offshore_generators.p_nom_min, + p_nom_min=offshore_generators.p_nom_min, + # p_nom_max= # Potentials + p_nom_extendable=offshore_generators.p_nom_extendable, + capital_cost=offshore_generators.capital_cost, + marginal_cost=costs.at["offwind", "marginal_cost"], + efficiency=costs.at["offwind", "efficiency"], + # p_max_pu= # PECD data + lifetime=costs.at["offwind", "lifetime"], + ) + + def add_offshore_electrolysers_tyndp( n: pypsa.Network, pyear: int, @@ -3176,7 +3239,9 @@ def add_offshore_grid_tyndp( # Add H2 pipeline connections offshore_grid_h2 = offshore_grid.query("carrier=='H2'").copy() - offshore_grid_h2.index = offshore_grid_h2.apply(make_index, axis=1, prefix="H2 pipeline") + offshore_grid_h2.index = offshore_grid_h2.apply( + make_index, axis=1, prefix="H2 pipeline" + ) offshore_grid_h2.loc[:, "capital_cost"] = ( annuity_factor.get("H2 (g) submarine pipeline") * offshore_grid_h2["capex"] + offshore_grid_h2["opex"] @@ -3201,8 +3266,9 @@ def add_offshore_grid_tyndp( def add_offshore_hubs_tyndp( n: pypsa.Network, pyear: int, - offshore_grid_fn: str, + offshore_generators_fn: str, offshore_electrolysers_fn: str, + offshore_grid_fn: str, costs: pd.DataFrame, spatial: spatial, logger: logging.Logger, @@ -3220,10 +3286,12 @@ def add_offshore_hubs_tyndp( The network object to add offshore hubs and grid to. pyear: int Planning horizon used to filter which reference grid data to include. - offshore_grid_fn : str - Path to the file containing offshore grid configuration data. + offshore_generators_fn : str + Path to the file containing offshore generators configuration data. offshore_electrolysers_fn : str Path to the file containing offshore electrolysers configuration data. + offshore_grid_fn : str + Path to the file containing offshore grid configuration data. costs : pd.DataFrame Technology costs assumptions. spatial : object, optional @@ -3272,6 +3340,7 @@ def add_offshore_hubs_tyndp( ) # Add power production units + add_offshore_generators_tyndp(n, pyear, offshore_generators_fn, logger) # Add H2 production units add_offshore_electrolysers_tyndp( @@ -7489,8 +7558,9 @@ def add_import_options( add_offshore_hubs_tyndp( n=n, pyear=int(snakemake.wildcards.planning_horizons), - offshore_grid_fn=snakemake.input.offshore_grid, + offshore_generators_fn=snakemake.input.offshore_generators, offshore_electrolysers_fn=snakemake.input.offshore_electrolysers, + offshore_grid_fn=snakemake.input.offshore_grid, costs=costs, spatial=spatial, logger=logger, From 4ed7dcf1e792db3a0eb14ca527a1db8edcee0630 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Jun 2025 15:35:58 +0000 Subject: [PATCH 042/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/clean_tyndp_h2_imports.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/clean_tyndp_h2_imports.py b/scripts/clean_tyndp_h2_imports.py index 2dad03e4d2..6ef03d3c53 100644 --- a/scripts/clean_tyndp_h2_imports.py +++ b/scripts/clean_tyndp_h2_imports.py @@ -116,7 +116,9 @@ def load_import_data(fn, countries_centroids): # Match countries centroids for defining coordinates of import nodes imports = match_centroids(imports, countries_centroids) - imports.index = imports.apply(make_index, axis=1, prefix="H2 import") + " - " + imports.Band + imports.index = ( + imports.apply(make_index, axis=1, prefix="H2 import") + " - " + imports.Band + ) return imports From 80859e4d6e39715c4040ea04d9da91b8f06c04c9 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 10 Jun 2025 15:45:21 +0200 Subject: [PATCH 043/165] doc: detail GENERATOR approach --- scripts/build_tyndp_offshore_hubs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 7b5a38954c..307b192386 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -227,6 +227,18 @@ def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int """ Load offshore generators data and format data. + The `COST` sheet provides techno-economic assumptions for offshore generators. + + The `EXISTING` sheet is assumed to contain the collected existing capacities collected prior to any reallocations intended to align with the PEMMDB. This sheet appears to be excluded from the modelling exercise. + + The `LAYER_POTENTIAL` sheet is viewed as containing the reallocated existing capacities and the theoretical potentials per technology. Existing capacities are specified for both electricity- and hydrogen-generating offshore wind farms. Technology shares from `EXISTING` will be used to supplement the data. + + The `ZONE_POTENTIAL` sheet is considered as the source for achievable potentials at each planning horizon. Technology shares from `LAYER_POTENTIAL` will be used to supplement the data. + + **Existing capacities** will be read from the `LAYER_POTENTIAL` sheet, utilizing technology shares specified in `EXISTING`. A discrepancy of 526 MW for `DEOH002` in 2045 (across all scenarios) is noted when comparing existing capacities with `ZONE_POTENTIAL`. It remains uncertain which of the two values is correct: 5828.55 MW from `LAYER_POTENTIAL` or 6354.55 MW from `ZONE_POTENTIAL`. + + **Potentials** will be read from the `ZONE_POTENTIAL` sheet, utilizing technology shares specified in `LAYER_POTENTIAL`. The same 526 MW discrepancy in `DEOH002` (across all planning horizons and scenarios) has been identified and needs to be addressed to ensure that existing capacities do not exceed their potential. + Parameters ---------- fn : str From 8b68c3a694358935cb588b84d743250fa45a85a8 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 10 Jun 2025 15:46:28 +0200 Subject: [PATCH 044/165] WIP feat: extend GENERATOR processing to take discrepancies into account --- scripts/build_tyndp_offshore_hubs.py | 136 +++++++++++++++++++++------ 1 file changed, 109 insertions(+), 27 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 307b192386..d356a5737c 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -8,6 +8,7 @@ import logging import geopandas as gpd +import numpy as np import pandas as pd from _helpers import configure_logging, set_scenario_config from shapely.geometry import Point @@ -255,62 +256,143 @@ def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int pd.DataFrame DataFrame containing the formatted offshore generators data. """ - column_dict = { + column_names = { "NODE": "location", "OFFSHORE_NODE": "bus0", "OFFSHORE_NODE_TYPE": "type", "YEAR": "pyear", "SCENARIO": "scenario", "TECHNOLOGY": "carrier", + "TECH1": "carrier", "CAPEX": "capex", "OPEX": "opex", "MW": "p_nom_min", + "EXISTING_MW": "p_nom_min", + "MAX_MW": "p_nom_max", } + column_del = [ + "TECH2", + "TECH3", + "TECH4", + "TECH5", + "TECH6", + "MARGIN_MW", + "LAYER", + ] + scenario_dict = { "Distributed Energy": "DE", "Global Ambition": "GA", "National Trends": "NT", } - # Load generators data + # Load data + def load_generators(sheet_name): + generators = ( + pd.read_excel( + fn, + sheet_name=sheet_name, + ) + .rename(columns=column_names) + .query("pyear in @planning_horizons") + .replace({"scenario": scenario_dict}) + .query("scenario == @scenario") + .assign( + carrier=lambda x: "offwind-" + + x.carrier.str.lower().replace("_", "-", regex=True) + ) + .drop(columns=column_del, errors="ignore") + ) + return generators + + generators_e = load_generators("EXISTING") + generators_l = load_generators("LAYER_POTENTIAL") + generators_z = load_generators("ZONE_POTENTIAL").drop( + columns=["carrier", "p_nom_min"] + ) + generators_c = load_generators("COST") + generators_c[["capex", "opex"]] = generators_c[["capex", "opex"]].mul( + 1e3 + ) # kEUR/MW to EUR/MW + + # Collect existing capacities in LAYER_POTENTIAL using H2 tech shares from EXISTING + def get_shares(df, values, dropna=True): + df_share = ( + df.pivot_table( + index=["bus0", "type", "pyear", "scenario"], + values=values, + columns="carrier", + ) + .pipe(lambda df: df.div(df.sum(axis=1), axis=0)) + .melt(ignore_index=False, value_name="tech_share") + .reset_index() + ) + if dropna: + df_share = df_share.dropna(subset="tech_share") + return df_share + + generators_e_share_raw = get_shares(generators_e, values="p_nom_min") + + generators_e_share = generators_e_share_raw.query( + "carrier.str.contains('h2')" + ).assign(carrier_rfc=lambda x: x.carrier.str.replace("h2", "dc", regex=True)) + + generators_e_share_rfc = ( + generators_e_share.drop(columns=["tech_share", "carrier"]) + .assign(carrier=lambda x: x.carrier_rfc) + .merge(generators_e_share_raw, how="left") + ) + generators_e_share = pd.concat([generators_e_share, generators_e_share_rfc]) + generators = ( - pd.read_excel( - fn, - sheet_name="EXISTING", + generators_l.merge( + generators_e_share, + how="outer", + left_on=["bus0", "type", "pyear", "scenario", "carrier"], + right_on=["bus0", "type", "pyear", "scenario", "carrier_rfc"], + suffixes=("_x", ""), ) - .rename(columns=column_dict) - .query("pyear in @planning_horizons") - .replace({"scenario": scenario_dict}) - .query("scenario == @scenario") .assign( - carrier=lambda x: "offwind-" - + x.carrier.str.lower().replace("_", "-", regex=True) + tech_share=lambda x: x.tech_share.fillna(1), + p_nom_min=lambda x: x.p_nom_min * x.tech_share, + carrier=lambda x: x.carrier.fillna(x.carrier_x), ) + .drop(columns=["carrier_x", "tech_share", "carrier_rfc", "p_nom_max"]) ) - # Load costs data - generators_costs = ( - pd.read_excel( - fn, - sheet_name="COST", + # Collect potentials in ZONE_POTENTIAL using tech shares from LAYER_POTENTIAL + generators_l_share = get_shares(generators_l, values="p_nom_max", dropna=False) + generators_z_tech = ( + generators_z.merge( + generators_l_share, + how="left", + on=["bus0", "type", "pyear", "scenario"], ) - .rename(columns=column_dict) - .query("pyear in @planning_horizons") - .replace({"scenario": scenario_dict}) - .query("scenario == @scenario") .assign( - carrier=lambda x: "offwind-" - + x.carrier.str.lower().replace("_", "-", regex=True) + tech_share=lambda x: x.tech_share.fillna(1), + p_nom_max=lambda x: x.p_nom_max * x.tech_share, ) + .drop(columns=["tech_share"]) + .query("p_nom_max != 0") ) - generators_costs[["capex", "opex"]] = generators_costs[["capex", "opex"]].mul( - 1e3 - ) # kEUR/MW to EUR/MW + generators = ( + generators.merge( + generators_z_tech, + how="outer", + on=["bus0", "type", "pyear", "scenario", "carrier"], + ) + .replace(0, np.nan) + .dropna(subset=["p_nom_min", "p_nom_max"], how="all") + ) - # Merge information - generators = generators_costs.merge( + generators[generators.p_nom_max.isna()] + + print("ok") + + # Collect costs assumptions + generators = generators_c.merge( generators, how="outer", on=["bus0", "location", "pyear", "scenario", "type", "carrier"], From 65bdca5addf93978f0ca3b71d935861c1a0b1ec8 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 10 Jun 2025 17:21:22 +0200 Subject: [PATCH 045/165] WIP feat: refactor everything --- scripts/build_tyndp_offshore_hubs.py | 199 ++++++++++++++++----------- 1 file changed, 118 insertions(+), 81 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index d356a5737c..0de8828ddb 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -224,11 +224,116 @@ def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[ return electrolysers +def get_shares(df, values, dropna=True): + df_share = ( + df.pivot_table( + index=["bus0", "type", "pyear", "scenario"], + values=values, + columns="carrier", + ) + .pipe(lambda df: df.div(df.sum(axis=1), axis=0)) + .melt(ignore_index=False, value_name="tech_share") + .reset_index() + ) + if dropna: + df_share = df_share.dropna(subset="tech_share") + return df_share + + +def collect_generators_capacities(generators_e, generators_l): + # Determine technology shares + generators_e_share_raw = get_shares(generators_e, values="p_nom_min") + + # Add missing H2 shares + generators_e_share = generators_e_share_raw.query( + "carrier.str.contains('h2')" + ).assign(carrier_rfc=lambda x: x.carrier.str.replace("h2", "dc", regex=True)) + + generators_e_share_rfc = ( + generators_e_share.drop(columns=["tech_share", "carrier"]) + .assign(carrier=lambda x: x.carrier_rfc) + .merge(generators_e_share_raw, how="left") + ) + generators_e_share = pd.concat([generators_e_share, generators_e_share_rfc]) + + # Compute existing capacities using shares + generators = ( + generators_l.merge( + generators_e_share, + how="outer", + left_on=["bus0", "type", "pyear", "scenario", "carrier"], + right_on=["bus0", "type", "pyear", "scenario", "carrier_rfc"], + suffixes=("_x", ""), + ) + .assign( + tech_share=lambda x: x.tech_share.fillna(1), + p_nom_min=lambda x: x.p_nom_min * x.tech_share, + carrier=lambda x: x.carrier.fillna(x.carrier_x), + ) + .drop(columns=["carrier_x", "tech_share", "carrier_rfc", "p_nom_max"]) + ) + + return generators + + +def collect_generators_potentials(generators, generators_l, generators_z): + # Get technology shares + generators_l_share = get_shares(generators_l, values="p_nom_max", dropna=False) + + # Compute potentials using shares + generators_z_tech = ( + generators_z.merge( + generators_l_share, + how="left", + on=["bus0", "type", "pyear", "scenario"], + ) + .assign( + tech_share=lambda x: x.tech_share.fillna(0), + p_nom_max=lambda x: x.p_nom_max * x.tech_share, + ) + .drop(columns=["tech_share"]) + .query("p_nom_max != 0") + ) + + # Integrate potentials in data + generators = ( + generators.merge( + generators_z_tech, + how="outer", + on=["bus0", "type", "pyear", "scenario", "carrier"], + ) + .replace(0, np.nan) + .dropna(subset=["p_nom_min", "p_nom_max"], how="all") + ) + + # Copy DC potentials for H2 wind farms + idx = generators.carrier.str.contains("h2") + generators_h2 = ( + generators.loc[idx] + .drop(columns="p_nom_max") + .assign( + carrier_ori=lambda x: x.carrier, + carrier=lambda x: x.carrier.str.replace("h2", "dc", regex=True), + ) + .merge( + generators.drop(columns="p_nom_min"), + how="left", + on=["bus0", "type", "pyear", "scenario", "carrier"], + ) + .drop(columns="carrier") + .rename(columns={"carrier_ori": "carrier"}) + ) + + generators = pd.concat([generators.loc[~idx], generators_h2]).fillna(0) + + return generators + + def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int]): """ Load offshore generators data and format data. - The `COST` sheet provides techno-economic assumptions for offshore generators. + The `COST` sheet provides techno-economic assumptions for offshore generators. It is assumed that only the specified generators can be expanded. The `EXISTING` sheet is assumed to contain the collected existing capacities collected prior to any reallocations intended to align with the PEMMDB. This sheet appears to be excluded from the modelling exercise. @@ -317,91 +422,23 @@ def load_generators(sheet_name): ) # kEUR/MW to EUR/MW # Collect existing capacities in LAYER_POTENTIAL using H2 tech shares from EXISTING - def get_shares(df, values, dropna=True): - df_share = ( - df.pivot_table( - index=["bus0", "type", "pyear", "scenario"], - values=values, - columns="carrier", - ) - .pipe(lambda df: df.div(df.sum(axis=1), axis=0)) - .melt(ignore_index=False, value_name="tech_share") - .reset_index() - ) - if dropna: - df_share = df_share.dropna(subset="tech_share") - return df_share - - generators_e_share_raw = get_shares(generators_e, values="p_nom_min") - - generators_e_share = generators_e_share_raw.query( - "carrier.str.contains('h2')" - ).assign(carrier_rfc=lambda x: x.carrier.str.replace("h2", "dc", regex=True)) - - generators_e_share_rfc = ( - generators_e_share.drop(columns=["tech_share", "carrier"]) - .assign(carrier=lambda x: x.carrier_rfc) - .merge(generators_e_share_raw, how="left") - ) - generators_e_share = pd.concat([generators_e_share, generators_e_share_rfc]) - - generators = ( - generators_l.merge( - generators_e_share, - how="outer", - left_on=["bus0", "type", "pyear", "scenario", "carrier"], - right_on=["bus0", "type", "pyear", "scenario", "carrier_rfc"], - suffixes=("_x", ""), - ) - .assign( - tech_share=lambda x: x.tech_share.fillna(1), - p_nom_min=lambda x: x.p_nom_min * x.tech_share, - carrier=lambda x: x.carrier.fillna(x.carrier_x), - ) - .drop(columns=["carrier_x", "tech_share", "carrier_rfc", "p_nom_max"]) - ) + generators = collect_generators_capacities(generators_e, generators_l) # Collect potentials in ZONE_POTENTIAL using tech shares from LAYER_POTENTIAL - generators_l_share = get_shares(generators_l, values="p_nom_max", dropna=False) - generators_z_tech = ( - generators_z.merge( - generators_l_share, - how="left", - on=["bus0", "type", "pyear", "scenario"], - ) - .assign( - tech_share=lambda x: x.tech_share.fillna(1), - p_nom_max=lambda x: x.p_nom_max * x.tech_share, - ) - .drop(columns=["tech_share"]) - .query("p_nom_max != 0") - ) + generators = collect_generators_potentials(generators, generators_l, generators_z) - generators = ( - generators.merge( - generators_z_tech, - how="outer", - on=["bus0", "type", "pyear", "scenario", "carrier"], - ) - .replace(0, np.nan) - .dropna(subset=["p_nom_min", "p_nom_max"], how="all") + # Collect cost assumptions + generators = generators.merge( + generators_c, + how="left", + on=["bus0", "pyear", "scenario", "type", "carrier"], ) - generators[generators.p_nom_max.isna()] - - print("ok") - - # Collect costs assumptions - generators = generators_c.merge( - generators, - how="outer", - on=["bus0", "location", "pyear", "scenario", "type", "carrier"], - ).assign(p_nom_min=lambda x: x["p_nom_min"].fillna(0)) - - # Assume non-extendable when missing data - # TODO Validate assumption - generators["p_nom_extendable"] = ~generators[["capex", "opex"]].isna().any(axis=1) - generators[["capex", "opex"]] = generators[["capex", "opex"]].fillna(0) + # Ensure that all cost assumptions are present + idx = generators[["capex", "opex"]].isna().any(axis=1) + if not (generators.loc[idx].p_nom_min == 0).all(): + raise RuntimeError("Missing generator cost data in input dataset.") + generators = generators.dropna(subset=["capex", "opex"]) # Rename UK in GB generators[["bus0", "location"]] = generators[["bus0", "location"]].replace( From 175d2757e264e4df66ae2b7e506c47077336bf58 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 11 Jun 2025 14:28:11 +0200 Subject: [PATCH 046/165] feat: extract both existing and potentials from layer and trajectories from zone --- rules/build_sector.smk | 1 + scripts/build_tyndp_offshore_hubs.py | 138 ++++++++++++--------------- scripts/prepare_sector_network.py | 4 +- 3 files changed, 64 insertions(+), 79 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index ba877ce59a..1ec0e6d7db 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1367,6 +1367,7 @@ if config["sector"]["offshore_hubs"]: offshore_grid=resources("offshore_grid.csv"), offshore_electrolysers=resources("offshore_electrolysers.csv"), offshore_generators=resources("offshore_generators.csv"), + offshore_zone_trajectories=resources("offshore_zone_trajectories.csv"), log: logs("build_tyndp_offshore_hubs.log"), benchmark: diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 0de8828ddb..e878899069 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -8,7 +8,6 @@ import logging import geopandas as gpd -import numpy as np import pandas as pd from _helpers import configure_logging, set_scenario_config from shapely.geometry import Point @@ -240,92 +239,70 @@ def get_shares(df, values, dropna=True): return df_share -def collect_generators_capacities(generators_e, generators_l): - # Determine technology shares - generators_e_share_raw = get_shares(generators_e, values="p_nom_min") - - # Add missing H2 shares - generators_e_share = generators_e_share_raw.query( - "carrier.str.contains('h2')" - ).assign(carrier_rfc=lambda x: x.carrier.str.replace("h2", "dc", regex=True)) +def collect_from_layer(generators_e, generators_l): + # Identify reallocations of radial wind farms using LAYER_POTENTIAL + idx = ["location", "bus0", "type", "pyear", "scenario", "carrier"] + generators_el = generators_e.merge( + generators_l, + how="outer", + on=["bus0", "type", "pyear", "scenario", "carrier"], + suffixes=("", "_l"), + ) - generators_e_share_rfc = ( - generators_e_share.drop(columns=["tech_share", "carrier"]) - .assign(carrier=lambda x: x.carrier_rfc) - .merge(generators_e_share_raw, how="left") + radial_inconsistent = generators_el.query( + "carrier.str.contains('-r') " # only radial connection + "and p_nom_min != p_nom_min_l " # when EXISTING and LAYER_POTENTIAL values are inconsistent + "and ~(p_nom_min.isna() and p_nom_min_l == 0)" # treat missing values and zeros as equivalent ) - generators_e_share = pd.concat([generators_e_share, generators_e_share_rfc]) - # Compute existing capacities using shares - generators = ( - generators_l.merge( - generators_e_share, - how="outer", - left_on=["bus0", "type", "pyear", "scenario", "carrier"], - right_on=["bus0", "type", "pyear", "scenario", "carrier_rfc"], - suffixes=("_x", ""), + # Fix EXISTING technologies by reallocating radial to hubs + generators_e_fixed = generators_e.copy().set_index(idx) + corrections = radial_inconsistent.assign( + location=lambda x: x.bus0, carrier=lambda x: x.carrier.str.replace("-r", "-oh") + ).set_index(idx) + generators_e_fixed = ( + pd.concat( + [ + generators_e_fixed.drop(radial_inconsistent.set_index(idx).index), + corrections[generators_e_fixed.columns], + ] ) - .assign( - tech_share=lambda x: x.tech_share.fillna(1), - p_nom_min=lambda x: x.p_nom_min * x.tech_share, - carrier=lambda x: x.carrier.fillna(x.carrier_x), - ) - .drop(columns=["carrier_x", "tech_share", "carrier_rfc", "p_nom_max"]) + .groupby(level=list(range(len(idx)))) + .sum() ) - return generators - - -def collect_generators_potentials(generators, generators_l, generators_z): - # Get technology shares - generators_l_share = get_shares(generators_l, values="p_nom_max", dropna=False) + # Calculate technology shares, considering PEMMDB related reallocations + tech_shares = get_shares(generators_e_fixed, values="p_nom_min") - # Compute potentials using shares - generators_z_tech = ( - generators_z.merge( - generators_l_share, - how="left", - on=["bus0", "type", "pyear", "scenario"], - ) + # Apply H2 to DC carrier mapping and get relevant shares + h2_shares = ( + tech_shares.query("carrier.str.contains('h2')") .assign( - tech_share=lambda x: x.tech_share.fillna(0), - p_nom_max=lambda x: x.p_nom_max * x.tech_share, + carrier_mapped=lambda x: x.carrier.str.replace( + "h2", "dc", regex=True + ).str.replace("-r", "-oh", regex=True) ) - .drop(columns=["tech_share"]) - .query("p_nom_max != 0") + .drop(columns=["tech_share", "carrier"]) + .merge(tech_shares, how="left") ) - # Integrate potentials in data + # Apply shares to data to calculate existing capacities generators = ( - generators.merge( - generators_z_tech, + generators_l.merge( + h2_shares, how="outer", - on=["bus0", "type", "pyear", "scenario", "carrier"], + left_on=["bus0", "type", "pyear", "scenario", "carrier"], + right_on=["bus0", "type", "pyear", "scenario", "carrier_mapped"], + suffixes=("_x", ""), ) - .replace(0, np.nan) - .dropna(subset=["p_nom_min", "p_nom_max"], how="all") - ) - - # Copy DC potentials for H2 wind farms - idx = generators.carrier.str.contains("h2") - generators_h2 = ( - generators.loc[idx] - .drop(columns="p_nom_max") .assign( - carrier_ori=lambda x: x.carrier, - carrier=lambda x: x.carrier.str.replace("h2", "dc", regex=True), - ) - .merge( - generators.drop(columns="p_nom_min"), - how="left", - on=["bus0", "type", "pyear", "scenario", "carrier"], + tech_share=lambda x: x.tech_share.fillna(1), + carrier=lambda x: x.carrier.fillna(x.carrier_x), + p_nom_min=lambda x: x.p_nom_min * x.tech_share, ) - .drop(columns="carrier") - .rename(columns={"carrier_ori": "carrier"}) + .drop(columns=["carrier_x", "tech_share", "carrier_mapped"]) ) - generators = pd.concat([generators.loc[~idx], generators_h2]).fillna(0) - return generators @@ -358,8 +335,13 @@ def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int Returns ------- - pd.DataFrame - DataFrame containing the formatted offshore generators data. + generators + pd.DataFrame + DataFrame containing the formatted offshore generators data. + + trajectories + pd.DataFrame + DataFrame containing the zone potentials trajectories """ column_names = { "NODE": "location", @@ -421,11 +403,11 @@ def load_generators(sheet_name): 1e3 ) # kEUR/MW to EUR/MW - # Collect existing capacities in LAYER_POTENTIAL using H2 tech shares from EXISTING - generators = collect_generators_capacities(generators_e, generators_l) + # Collect existing capacities and potentials in LAYER_POTENTIAL using H2 tech shares from EXISTING + generators = collect_from_layer(generators_e, generators_l) - # Collect potentials in ZONE_POTENTIAL using tech shares from LAYER_POTENTIAL - generators = collect_generators_potentials(generators, generators_l, generators_z) + # Collect potentials trajectories in ZONE_POTENTIAL + zone_trajectories = generators_z # Collect cost assumptions generators = generators.merge( @@ -439,13 +421,14 @@ def load_generators(sheet_name): if not (generators.loc[idx].p_nom_min == 0).all(): raise RuntimeError("Missing generator cost data in input dataset.") generators = generators.dropna(subset=["capex", "opex"]) + generators.loc[:, "p_nom_extendable"] = True # Rename UK in GB generators[["bus0", "location"]] = generators[["bus0", "location"]].replace( "UK", "GB", regex=True ) - return generators + return generators, zone_trajectories if __name__ == "__main__": @@ -473,7 +456,7 @@ def load_generators(sheet_name): snakemake.input.electrolysers, snakemake.params["scenario"], planning_horizons ) - generators = load_offshore_generators( + generators, zone_trajectories = load_offshore_generators( snakemake.input.generators, snakemake.params["scenario"], planning_horizons ) @@ -482,3 +465,4 @@ def load_generators(sheet_name): grid.to_csv(snakemake.output.offshore_grid, index=False) electrolysers.to_csv(snakemake.output.offshore_electrolysers, index=False) generators.to_csv(snakemake.output.offshore_generators, index=False) + zone_trajectories.to_csv(snakemake.output.offshore_zone_trajectories, index=False) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 795e3d330a..f5b25c26b5 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3082,7 +3082,7 @@ def add_offshore_generators_tyndp( carrier=offshore_generators.carrier, p_nom=offshore_generators.p_nom_min, p_nom_min=offshore_generators.p_nom_min, - # p_nom_max= # Potentials + p_nom_max=offshore_generators.p_nom_min, p_nom_extendable=offshore_generators.p_nom_extendable, capital_cost=offshore_generators.capital_cost, marginal_cost=costs.at["offwind", "marginal_cost"], @@ -3270,7 +3270,7 @@ def add_offshore_hubs_tyndp( offshore_electrolysers_fn: str, offshore_grid_fn: str, costs: pd.DataFrame, - spatial: spatial, + spatial: SimpleNamespace, logger: logging.Logger, nyears: float = 1, ): From 40f288f639b377119519fe7b6f5a5356e6914367 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 11 Jun 2025 16:42:21 +0200 Subject: [PATCH 047/165] doc: improve documentation of load_offshore_generators --- scripts/build_tyndp_offshore_hubs.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index e878899069..7549b6c327 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -310,17 +310,15 @@ def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int """ Load offshore generators data and format data. - The `COST` sheet provides techno-economic assumptions for offshore generators. It is assumed that only the specified generators can be expanded. + The `EXISTING` sheet is assumed to contain the collected existing capacities collected prior to any reallocations intended to align with the PEMMDB. This sheet appears to be excluded from the modelling exercise, except for hydrogen-generating capacities. - The `EXISTING` sheet is assumed to contain the collected existing capacities collected prior to any reallocations intended to align with the PEMMDB. This sheet appears to be excluded from the modelling exercise. + The `LAYER_POTENTIAL` sheet is viewed as containing the reallocated existing capacities (excluding hydrogen-generating specific information) and the theoretical potentials per technology. Existing capacities are specified for both electricity- and hydrogen-generating offshore wind farms. Technology shares from `EXISTING` will be used to supplement the data. - The `LAYER_POTENTIAL` sheet is viewed as containing the reallocated existing capacities and the theoretical potentials per technology. Existing capacities are specified for both electricity- and hydrogen-generating offshore wind farms. Technology shares from `EXISTING` will be used to supplement the data. + The `ZONE_POTENTIAL` sheet is considered as the source for achievable potentials for each node across all planning horizons. It establishes a nodal constraint on top of the theoretical potentials outlined by `LAYER_POTENTIAL`. - The `ZONE_POTENTIAL` sheet is considered as the source for achievable potentials at each planning horizon. Technology shares from `LAYER_POTENTIAL` will be used to supplement the data. + **Existing capacities** will be read from the `LAYER_POTENTIAL` sheet, utilizing technology shares specified in `EXISTING` for hydrogen-generating capacities. A discrepancy of 526 MW for `DEOH002` in 2045 (across all scenarios) is noted when comparing existing capacities with `ZONE_POTENTIAL`. It remains uncertain which of the two values is correct: 5828.55 MW from `LAYER_POTENTIAL` or 6354.55 MW from `ZONE_POTENTIAL`. - **Existing capacities** will be read from the `LAYER_POTENTIAL` sheet, utilizing technology shares specified in `EXISTING`. A discrepancy of 526 MW for `DEOH002` in 2045 (across all scenarios) is noted when comparing existing capacities with `ZONE_POTENTIAL`. It remains uncertain which of the two values is correct: 5828.55 MW from `LAYER_POTENTIAL` or 6354.55 MW from `ZONE_POTENTIAL`. - - **Potentials** will be read from the `ZONE_POTENTIAL` sheet, utilizing technology shares specified in `LAYER_POTENTIAL`. The same 526 MW discrepancy in `DEOH002` (across all planning horizons and scenarios) has been identified and needs to be addressed to ensure that existing capacities do not exceed their potential. + **Potentials** will be obtained from both the `LAYER_POTENTIAL` and the `ZONE_POTENTIAL` sheets. `LAYER_POTENTIAL` will establish a technology level constraint, while `ZONE_POTENTIAL` will restrict expansion across all technologies at each node. The same 526 MW discrepancy in `DEOH002` (across all planning horizons and scenarios) has been noted and needs to be addressed to ensure that existing capacities do not exceed their potential. Parameters ---------- @@ -335,13 +333,11 @@ def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int Returns ------- - generators - pd.DataFrame - DataFrame containing the formatted offshore generators data. + generators : pd.DataFrame + DataFrame containing the formatted offshore generators data - trajectories - pd.DataFrame - DataFrame containing the zone potentials trajectories + zone_trajectories : pd.DataFrame + DataFrame containing the zone potentials trajectories """ column_names = { "NODE": "location", From e1582e4f6e9d93010873e340c564608f0b6c129a Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 11 Jun 2025 17:07:47 +0200 Subject: [PATCH 048/165] feat: resolve discrepancy in DEOH002 --- scripts/build_tyndp_offshore_hubs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 7549b6c327..435496abad 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -316,9 +316,9 @@ def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int The `ZONE_POTENTIAL` sheet is considered as the source for achievable potentials for each node across all planning horizons. It establishes a nodal constraint on top of the theoretical potentials outlined by `LAYER_POTENTIAL`. - **Existing capacities** will be read from the `LAYER_POTENTIAL` sheet, utilizing technology shares specified in `EXISTING` for hydrogen-generating capacities. A discrepancy of 526 MW for `DEOH002` in 2045 (across all scenarios) is noted when comparing existing capacities with `ZONE_POTENTIAL`. It remains uncertain which of the two values is correct: 5828.55 MW from `LAYER_POTENTIAL` or 6354.55 MW from `ZONE_POTENTIAL`. + **Existing capacities** will be read from the `LAYER_POTENTIAL` sheet, utilizing technology shares specified in `EXISTING` for hydrogen-generating capacities. A discrepancy of 526 MW for `DEOH002` in 2045 (across all scenarios) is noted when comparing existing capacities with `ZONE_POTENTIAL`. It remains uncertain which of the two values is correct: 5828.55 MW from `LAYER_POTENTIAL` or 6354.55 MW from `ZONE_POTENTIAL`. Currently, the value of 5828.55 MW is used. - **Potentials** will be obtained from both the `LAYER_POTENTIAL` and the `ZONE_POTENTIAL` sheets. `LAYER_POTENTIAL` will establish a technology level constraint, while `ZONE_POTENTIAL` will restrict expansion across all technologies at each node. The same 526 MW discrepancy in `DEOH002` (across all planning horizons and scenarios) has been noted and needs to be addressed to ensure that existing capacities do not exceed their potential. + **Potentials** will be obtained from both the `LAYER_POTENTIAL` and the `ZONE_POTENTIAL` sheets. `LAYER_POTENTIAL` will establish a technology level constraint, while `ZONE_POTENTIAL` will restrict expansion across all technologies at each node. The same 526 MW discrepancy in `DEOH002` (across all planning horizons and scenarios) has been noted and needs to be addressed to ensure that existing capacities do not exceed their potential. Currently, the value `ZONE_POTENTIAL` value is corrected at 5828.55 MW. Parameters ---------- @@ -405,6 +405,12 @@ def load_generators(sheet_name): # Collect potentials trajectories in ZONE_POTENTIAL zone_trajectories = generators_z + # Resolve discrepancy in DEOH002 + idx = zone_trajectories.query("bus0=='DEOH002' and pyear in [2045, 2050]").index + zone_trajectories.loc[idx, "p_nom_max"] = ( + zone_trajectories.loc[idx, "p_nom_max"] - 526 + ) + # Collect cost assumptions generators = generators.merge( generators_c, From 14f5615f30b0ae1d36afabf69f7cb63a8e856c35 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 11 Jun 2025 17:39:09 +0200 Subject: [PATCH 049/165] feat: filter for selected countries --- rules/build_sector.smk | 1 + scripts/build_tyndp_offshore_hubs.py | 75 ++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1ec0e6d7db..e8d26465a4 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1355,6 +1355,7 @@ if config["sector"]["offshore_hubs"]: params: planning_horizons=config_provider("scenario", "planning_horizons"), scenario=config_provider("tyndp_scenario"), + countries=config_provider("countries"), input: nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 435496abad..d7b28cfb06 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -17,7 +17,7 @@ GEO_CRS = "EPSG:4326" -def load_offshore_hubs(fn: str): +def load_offshore_hubs(fn: str, countries: list[str]): """ Load offshore hubs coordinates and format data. @@ -27,6 +27,8 @@ def load_offshore_hubs(fn: str): ---------- fn : str Path to the Excel file containing offshore hub data. + countries : list[str] + List of country codes used to clean data. Returns ------- @@ -50,14 +52,18 @@ def load_offshore_hubs(fn: str): mask = nodes["Bus"].str.contains("OH") nodes.loc[mask, "location"] = nodes.loc[mask, "Bus"] + nodes.loc[:, "country"] = nodes.loc[:, "location"].str[:2] nodes["geometry"] = nodes.apply(lambda row: Point(row["x"], row["y"]), axis=1) nodes = gpd.GeoDataFrame(nodes, geometry="geometry", crs=GEO_CRS) # rename UK in GB - nodes[["Bus", "location"]] = nodes[["Bus", "location"]].replace( - "UK", "GB", regex=True - ) + nodes[["Bus", "location", "country"]] = nodes[ + ["Bus", "location", "country"] + ].replace("UK", "GB", regex=True) + + # filter selected countries + nodes = nodes.query("country in @countries") return nodes @@ -73,7 +79,11 @@ def expand_all_scenario(df: pd.DataFrame, scenarios: list): def load_offshore_grid( - fn: str, nodes: pd.DataFrame, scenario: str, planning_horizons: list[int] + fn: str, + nodes: pd.DataFrame, + scenario: str, + planning_horizons: list[int], + countries: list[str], ): """ Load offshore grid (electricity and hydrogen) and format data. @@ -91,6 +101,8 @@ def load_offshore_grid( "NT" (National Trends). planning_horizons : list[int] List of planning years to include in the cost data filtering. + countries : list[str] + List of country codes used to clean data. Returns ------- @@ -159,10 +171,18 @@ def load_offshore_grid( # Rename UK in GB grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB", regex=True) + # Filter selected countries + grid = grid.assign( + country0=lambda x: x.bus0.str[:2], + country1=lambda x: x.bus1.str[:2], + ).query("country0 in @countries and country1 in @countries") + return grid -def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[int]): +def load_offshore_electrolysers( + fn: str, scenario: str, planning_horizons: list[int], countries: list[str] +): """ Load offshore electrolysers data and format data. @@ -176,6 +196,8 @@ def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[ "NT" (National Trends). planning_horizons : list[int] List of planning years to include in the cost data filtering. + countries : list[str] + List of country codes used to clean data. Returns ------- @@ -220,6 +242,11 @@ def load_offshore_electrolysers(fn: str, scenario: str, planning_horizons: list[ ["bus0", "bus1", "location"] ].replace("UK", "GB", regex=True) + # filter selected countries + electrolysers = electrolysers.assign(country=lambda x: x.bus0.str[:2]).query( + "country in @countries" + ) + return electrolysers @@ -306,7 +333,9 @@ def collect_from_layer(generators_e, generators_l): return generators -def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int]): +def load_offshore_generators( + fn: str, scenario: str, planning_horizons: list[int], countries: list[str] +): """ Load offshore generators data and format data. @@ -330,6 +359,8 @@ def load_offshore_generators(fn: str, scenario: str, planning_horizons: list[int "NT" (National Trends). planning_horizons : list[int] List of planning years to include in the cost data filtering. + countries : list[str] + List of country codes used to clean data. Returns ------- @@ -429,6 +460,17 @@ def load_generators(sheet_name): generators[["bus0", "location"]] = generators[["bus0", "location"]].replace( "UK", "GB", regex=True ) + zone_trajectories["bus0"] = zone_trajectories["bus0"].replace( + "UK", "GB", regex=True + ) + + # Filter selected countries + generators = generators.assign(country=lambda x: x.bus0.str[:2]).query( + "country in @countries" + ) + zone_trajectories = zone_trajectories.assign( + country=lambda x: x.bus0.str[:2] + ).query("country in @countries") return generators, zone_trajectories @@ -447,19 +489,30 @@ def load_generators(sheet_name): # Parameters scenario = snakemake.params["scenario"] planning_horizons = snakemake.params["planning_horizons"] + countries = snakemake.params["countries"] - nodes = load_offshore_hubs(snakemake.input.nodes) + nodes = load_offshore_hubs(snakemake.input.nodes, countries) grid = load_offshore_grid( - snakemake.input.grid, nodes, snakemake.params["scenario"], planning_horizons + snakemake.input.grid, + nodes, + snakemake.params["scenario"], + planning_horizons, + countries, ) electrolysers = load_offshore_electrolysers( - snakemake.input.electrolysers, snakemake.params["scenario"], planning_horizons + snakemake.input.electrolysers, + snakemake.params["scenario"], + planning_horizons, + countries, ) generators, zone_trajectories = load_offshore_generators( - snakemake.input.generators, snakemake.params["scenario"], planning_horizons + snakemake.input.generators, + snakemake.params["scenario"], + planning_horizons, + countries, ) # Save data From 08b497fe029c42f06e548ab79fb6b50e94fab725 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 12 Jun 2025 13:33:27 +0200 Subject: [PATCH 050/165] fix: take efficiency into account for h2-generating wind farms --- scripts/prepare_sector_network.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index f5b25c26b5..f0822278b0 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3033,6 +3033,7 @@ def add_offshore_generators_tyndp( n: pypsa.Network, pyear: int, offshore_generators_fn: str, + costs: pd.DataFrame, logger: logging.Logger, ): """ @@ -3041,6 +3042,9 @@ def add_offshore_generators_tyndp( This function adds offshore generation capacity, various offshore wind turbines (AC and H2), to the offshore hub buses in the network. + Existing capacities and potentials are adjusted for hydrogen-generating wind farms + to account for efficiency. + Parameters ---------- n : pypsa.Network @@ -3049,6 +3053,8 @@ def add_offshore_generators_tyndp( Planning horizon used to filter which reference generator data to include. offshore_generators : str Path to the file containing offshore generators configuration data. + costs : pd.DataFrame + Technology costs assumptions. logger : logging.Logger Logger for output messages. If None, no logging is performed. @@ -3069,6 +3075,16 @@ def add_offshore_generators_tyndp( offshore_generators.bus0 + " " + offshore_generators.carrier ) + h2_idx = offshore_generators.filter(like="h2", axis=0).index + offshore_generators.loc[h2_idx, ["p_nom_min", "p_nom_max"]] = ( + offshore_generators.loc[h2_idx, ["p_nom_min", "p_nom_max"]].mul( + costs.at["electrolysis", "efficiency"] + ) + ) + offshore_generators.loc[h2_idx, ["capex", "opex"]] = offshore_generators.loc[ + h2_idx, ["capex", "opex"] + ].div(costs.at["electrolysis", "efficiency"]) + annuity_factor = calculate_annuity(costs["lifetime"], costs["discount rate"]) offshore_generators.loc[:, "capital_cost"] = ( annuity_factor.get("electrolysis") * offshore_generators["capex"] @@ -3340,7 +3356,7 @@ def add_offshore_hubs_tyndp( ) # Add power production units - add_offshore_generators_tyndp(n, pyear, offshore_generators_fn, logger) + add_offshore_generators_tyndp(n, pyear, offshore_generators_fn, costs, logger) # Add H2 production units add_offshore_electrolysers_tyndp( From 127af80cf669d1fd889cb50c866e8b21b691c0fd Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 12 Jun 2025 17:04:15 +0200 Subject: [PATCH 051/165] feat: attach PECD profiles to offwind generators --- scripts/prepare_sector_network.py | 42 +++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index f0822278b0..dc1d6e12a1 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3033,6 +3033,7 @@ def add_offshore_generators_tyndp( n: pypsa.Network, pyear: int, offshore_generators_fn: str, + profiles: dict[str, str], costs: pd.DataFrame, logger: logging.Logger, ): @@ -3053,6 +3054,9 @@ def add_offshore_generators_tyndp( Planning horizon used to filter which reference generator data to include. offshore_generators : str Path to the file containing offshore generators configuration data. + profiles : dict[str, str] + Dictionary mapping technology names to profile file paths + e.g. {'offwind-dc': 'path/to/profile.nc'} costs : pd.DataFrame Technology costs assumptions. logger : logging.Logger @@ -3067,6 +3071,7 @@ def add_offshore_generators_tyndp( offshore_generators = pd.read_csv(offshore_generators_fn).query("pyear==@pyear") + # Assign locations and index mask = offshore_generators["carrier"].str.contains("h2") offshore_generators.loc[mask, ["bus0", "location"]] = ( offshore_generators.loc[mask, ["bus0", "location"]] + " H2" @@ -3075,6 +3080,7 @@ def add_offshore_generators_tyndp( offshore_generators.bus0 + " " + offshore_generators.carrier ) + # Adjust capacities and costs to account for efficiency h2_idx = offshore_generators.filter(like="h2", axis=0).index offshore_generators.loc[h2_idx, ["p_nom_min", "p_nom_max"]] = ( offshore_generators.loc[h2_idx, ["p_nom_min", "p_nom_max"]].mul( @@ -3085,12 +3091,37 @@ def add_offshore_generators_tyndp( h2_idx, ["capex", "opex"] ].div(costs.at["electrolysis", "efficiency"]) + # Determine capital_cost annuity_factor = calculate_annuity(costs["lifetime"], costs["discount rate"]) offshore_generators.loc[:, "capital_cost"] = ( annuity_factor.get("electrolysis") * offshore_generators["capex"] + offshore_generators["opex"] ) * nyears + # Load PECD profiles + p_max_pu = [] + for key, fn in profiles.items(): + tech = key[len("profile_pecd_") :] + techs = offshore_generators[ + offshore_generators.carrier.str.contains(tech) + ].carrier.unique() + techs = ["H2 " + tech_i if "h2" in tech_i else tech_i for tech_i in techs] + + with xr.open_dataset(fn) as ds: + ds = ds.sel( + year=pyear, bin=0, time=n.snapshots, drop=True + ) # ToDo Remove time sel once sns are filtered in PECD data + p_max_pu_i = ds["profile"].to_pandas() + + for tech_i in techs: + p_max_pu.append(p_max_pu_i.rename(columns=lambda x: x + " " + tech_i)) + + p_max_pu = pd.concat(p_max_pu, axis=1) + p_max_pu = p_max_pu.reindex( + offshore_generators.index, axis=1, fill_value=0 + ) # ToDo Simplify once missing nodes are addressed in PECD data + + # Add generators to the network n.add( "Generator", offshore_generators.index, @@ -3103,7 +3134,7 @@ def add_offshore_generators_tyndp( capital_cost=offshore_generators.capital_cost, marginal_cost=costs.at["offwind", "marginal_cost"], efficiency=costs.at["offwind", "efficiency"], - # p_max_pu= # PECD data + p_max_pu=p_max_pu, lifetime=costs.at["offwind", "lifetime"], ) @@ -3285,6 +3316,7 @@ def add_offshore_hubs_tyndp( offshore_generators_fn: str, offshore_electrolysers_fn: str, offshore_grid_fn: str, + profiles: dict[str, str], costs: pd.DataFrame, spatial: SimpleNamespace, logger: logging.Logger, @@ -3308,6 +3340,9 @@ def add_offshore_hubs_tyndp( Path to the file containing offshore electrolysers configuration data. offshore_grid_fn : str Path to the file containing offshore grid configuration data. + profiles : dict[str, str] + Dictionary mapping technology names to profile file paths + e.g. {'offwind-dc': 'path/to/profile.nc'} costs : pd.DataFrame Technology costs assumptions. spatial : object, optional @@ -3356,7 +3391,9 @@ def add_offshore_hubs_tyndp( ) # Add power production units - add_offshore_generators_tyndp(n, pyear, offshore_generators_fn, costs, logger) + add_offshore_generators_tyndp( + n, pyear, offshore_generators_fn, profiles, costs, logger + ) # Add H2 production units add_offshore_electrolysers_tyndp( @@ -7577,6 +7614,7 @@ def add_import_options( offshore_generators_fn=snakemake.input.offshore_generators, offshore_electrolysers_fn=snakemake.input.offshore_electrolysers, offshore_grid_fn=snakemake.input.offshore_grid, + profiles=profiles, costs=costs, spatial=spatial, logger=logger, From 4cdfe9dd50511ba97bddd93d121ee21aada694c6 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 12 Jun 2025 17:33:38 +0200 Subject: [PATCH 052/165] refactor: use tyndp specific renewable carriers and differentiate between tyndp renewable carriers and pecd renewable profiles --- config/config.default.yaml | 14 +++++--- config/config.tyndp.yaml | 16 ++++++--- config/test/config.tyndp.yaml | 16 ++++++--- doc/configtables/electricity.csv | 5 +-- rules/build_electricity.smk | 62 +++++++++++++++++--------------- rules/build_sector.smk | 17 +++++++-- rules/solve_myopic.smk | 4 +-- scripts/add_brownfield.py | 21 ++++++++--- scripts/add_electricity.py | 20 +++++++---- scripts/add_existing_baseyear.py | 34 ++++++++++-------- scripts/clean_pecd_data.py | 1 + 11 files changed, 138 insertions(+), 72 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 234b0ecabc..23c5ddfa14 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -111,12 +111,18 @@ electricity: conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float, hydro] - tyndp_renewable_profiles: + pecd_renewable_profiles: enable: false technologies: - - offwind-ac - - offwind-dc - - offwind-float + offwind: + - offwind-ac-fb-r + - offwind-ac-fl-r + - offwind-dc-fb-r + - offwind-dc-fl-r + - offwind-dc-fb-oh + - offwind-dc-fl-oh + - offwind-h2-fb-oh + - offwind-h2-fl-oh estimate_renewable_capacities: enable: true diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 8a0836ec6c..6f6514d272 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -38,12 +38,20 @@ co2_budget: electricity: base_network: tyndp-raw transmission_limit: v1.0 - tyndp_renewable_profiles: + pecd_renewable_profiles: enable: true technologies: - - offwind-ac - - offwind-dc - - offwind-float + offwind: + - offwind-ac-fb-r + - offwind-ac-fl-r + - offwind-dc-fb-r + - offwind-dc-fl-r + - offwind-dc-fb-oh + - offwind-dc-fl-oh + - offwind-h2-fb-oh + - offwind-h2-fl-oh + + renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro] links: p_max_pu: 1.0 diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 4cbd874a44..797edd2c3c 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -35,12 +35,18 @@ co2_budget: electricity: base_network: tyndp-raw transmission_limit: v1.0 - tyndp_renewable_profiles: + pecd_renewable_profiles: enable: true technologies: - - offwind-ac - - offwind-dc - - offwind-float + offwind: + - offwind-ac-fb-r + - offwind-ac-fl-r + - offwind-dc-fb-r + - offwind-dc-fl-r + - offwind-dc-fb-oh + - offwind-dc-fl-oh + - offwind-h2-fb-oh + - offwind-h2-fl-oh extendable_carriers: Generator: [OCGT] @@ -48,7 +54,7 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float] + renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro] atlite: default_cutout: europe-2013-03-sarah3-era5 diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 0a738bc50e..3af8782456 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -30,9 +30,10 @@ everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignit conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound." ,,, renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. -tyndp_renewable_profiles,,, +pecd_renewable_profiles,,, -- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. --- technologies,--,list,Select the renewable technologies for which PECD renewable profiles are used. Carriers not listed use default renewable profiles. +-- technologies,,, +-- -- {pecd_profile},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. Carriers not listed use default renewable profiles. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 8c2bba23cd..9c342da7ad 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -381,6 +381,29 @@ rule build_renewable_profiles: "../scripts/build_renewable_profiles.py" +rule clean_pecd_data: + params: + scenario=config_provider("tyndp_scenario"), + snapshots=config_provider("snapshots"), + input: + offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", + onshore_buses=resources("busmap_base_s_all.csv"), + fn_pecd="data/tyndp_2024_bundle/PECD", + output: + pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), + log: + logs("clean_pecd_data_{technology}_{planning_horizons}.log"), + benchmark: + benchmarks("clean_pecd_data_{technology}_{planning_horizons}") + threads: 4 + resources: + mem_mb=4000, + conda: + "../envs/environment.yaml" + script: + "../scripts/clean_pecd_data.py" + + def input_data_pecd(w): return { f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv") @@ -417,29 +440,6 @@ rule build_renewable_profiles_pecd: "../scripts/build_renewable_profiles_pecd.py" -rule clean_pecd_data: - params: - scenario=config_provider("tyndp_scenario"), - snapshots=config_provider("snapshots"), - input: - offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", - onshore_buses=resources("busmap_base_s_all.csv"), - fn_pecd="data/tyndp_2024_bundle/PECD", - output: - pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), - log: - logs("clean_pecd_data_{technology}_{planning_horizons}.log"), - benchmark: - benchmarks("clean_pecd_data_{technology}_{planning_horizons}") - threads: 4 - resources: - mem_mb=4000, - conda: - "../envs/environment.yaml" - script: - "../scripts/clean_pecd_data.py" - - rule build_monthly_prices: input: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", @@ -599,7 +599,7 @@ def input_class_regions(w): ) for tech in set(config_provider("electricity", "renewable_carriers")(w)) - {"hydro"} - - set(tyndp_renewable_profiles(w)) + - set(tyndp_renewable_carriers(w)) } @@ -765,10 +765,16 @@ rule cluster_network: "../scripts/cluster_network.py" -def tyndp_renewable_profiles(w): +def tyndp_renewable_carriers(w): return ( - config_provider("electricity", "tyndp_renewable_profiles", "technologies")(w) - if config_provider("electricity", "tyndp_renewable_profiles", "enable")(w) + [ + subcarrier + for carrier in config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w).values() + for subcarrier in carrier + ] + if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) else [] ) @@ -781,7 +787,7 @@ def input_profile_tech(w): else f"profile_{tech}.nc" ) for tech in set(config_provider("electricity", "renewable_carriers")(w)) - - set(tyndp_renewable_profiles(w)) + - set(tyndp_renewable_carriers(w)) } diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 8088ee2262..9853ba5ccf 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1226,14 +1226,27 @@ def input_profile_offwind(w): f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") for tech in ["offwind-ac", "offwind-dc", "offwind-float"] if (tech in config_provider("electricity", "renewable_carriers")(w)) - and (tech not in tyndp_renewable_profiles(w)) + and (tech not in tyndp_renewable_carriers(w)) } +def pecd_renewable_profiles(w): + return ( + [ + carrier + for carrier in config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w).keys() + ] + if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) + else [] + ) + + def input_profile_pecd(w): return { f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") - for tech in tyndp_renewable_profiles(w) + for tech in pecd_renewable_profiles(w) } diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 692785fa7c..ac721e8d2c 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -62,7 +62,7 @@ def input_profile_tech_brownfield(w): f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") for tech in ( set(config_provider("electricity", "renewable_carriers")(w)) - - set(tyndp_renewable_profiles(w)) + - set(tyndp_renewable_carriers(w)) ) if tech != "hydro" } @@ -71,7 +71,7 @@ def input_profile_tech_brownfield(w): def input_profile_tech_brownfied_pecd(w): return { f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") - for tech in tyndp_renewable_profiles(w) + for tech in pecd_renewable_profiles(w) } diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index cf4eb621bf..4c8512980a 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -218,10 +218,23 @@ def adjust_renewable_profiles(n, input_profiles, params, year): pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill() ) - # TODO: hotfix remove filter for tyndp_renewable_profiles after tyndp generators are added - for carrier in set(params["carriers"]) - set( - params["electricity"]["tyndp_renewable_profiles"]["technologies"] - ): + # TODO: hotfix remove filter for tyndp_renewable_carriers after tyndp generators are added + tyndp_renewable_carriers = ( + [ + subcarrier + for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ + "technologies" + ].values() + for subcarrier in carrier + ] + if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] + else [] + ) + if len(tyndp_renewable_carriers) > 0: + logger.info( + f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." + ) + for carrier in set(params["carriers"]) - set(tyndp_renewable_carriers): if carrier == "hydro": continue diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index d7c9a314cb..176f6c0a1c 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1203,20 +1203,26 @@ def attach_stores( params.link_length_factor, ) - tyndp_renewable_profiles = ( - params.electricity["tyndp_renewable_profiles"]["technologies"] - if params.electricity["tyndp_renewable_profiles"]["enable"] + tyndp_renewable_carriers = ( + [ + subcarrier + for carrier in params.electricity["pecd_renewable_profiles"][ + "technologies" + ].values() + for subcarrier in carrier + ] + if params.electricity["pecd_renewable_profiles"]["enable"] else [] ) - if len(tyndp_renewable_profiles) > 0: + if len(tyndp_renewable_carriers) > 0: logger.info( - f"Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'. They will be attached later on with TYNDP data." + f"Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'. They will be attached later on with TYNDP data." ) renewable_carriers = set( [ carrier for carrier in params.electricity["renewable_carriers"] - if carrier not in tyndp_renewable_profiles + if carrier not in tyndp_renewable_carriers ] ) extendable_carriers = params.electricity["extendable_carriers"] @@ -1285,7 +1291,7 @@ def attach_stores( tech_map = { key: value for key, value in estimate_renewable_caps["technology_mapping"].items() - if value not in tyndp_renewable_profiles + if value not in tyndp_renewable_carriers } expansion_limit = estimate_renewable_caps["expansion_limit"] year = estimate_renewable_caps["year"] diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index d2bf9aac1b..d14ad9a850 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -68,7 +68,7 @@ def add_existing_renewables( costs: pd.DataFrame, df_agg: pd.DataFrame, countries: list[str], - tyndp_renewable_profiles: list[str], + tyndp_renewable_carriers: list[str], ) -> None: """ Add existing renewable capacities to conventional power plant data. @@ -83,8 +83,8 @@ def add_existing_renewables( Network containing topology and generator data countries : list List of country codes to consider - tyndp_renewable_profiles: list - List of renewable profile technologies taken from tyndp PECD + tyndp_renewable_carriers: list + List of renewable technologies from TYNDP Returns ------- @@ -93,11 +93,11 @@ def add_existing_renewables( """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} # TODO: remove when TYNDP renewable generators are added - if len(tyndp_renewable_profiles) > 0: + if len(tyndp_renewable_carriers) > 0: logger.info( - f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_profiles)}'." + f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." ) - for k in tyndp_renewable_profiles: + for k in tyndp_renewable_carriers: tech_map.pop(k, None) irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() @@ -159,7 +159,7 @@ def add_power_capacities_installed_before_baseyear( countries: list[str], capacity_threshold: float, lifetime_values: dict[str, float], - tyndp_renewable_profiles: list[str], + tyndp_renewable_carriers: list[str], ) -> None: """ Add power generation capacities installed before base year. @@ -182,8 +182,8 @@ def add_power_capacities_installed_before_baseyear( Minimum capacity threshold lifetime_values : dict Default values for missing data - tyndp_renewable_profiles: list - List of renewable profile technologies taken from tyndp PECD + tyndp_renewable_carriers: list + List of renewable technologies from TYNDP """ logger.debug(f"Adding power capacities installed before {baseyear}") @@ -236,7 +236,7 @@ def add_power_capacities_installed_before_baseyear( costs=costs, n=n, countries=countries, - tyndp_renewable_profiles=tyndp_renewable_profiles, + tyndp_renewable_carriers=tyndp_renewable_carriers, ) # drop assets which are already phased out / decommissioned phased_out = df_agg[df_agg["DateOut"] < baseyear].index @@ -741,9 +741,15 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector - tyndp_renewable_profiles = ( - snakemake.params.electricity["tyndp_renewable_profiles"]["technologies"] - if snakemake.params.electricity["tyndp_renewable_profiles"]["enable"] + tyndp_renewable_carriers = ( + [ + subcarrier + for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ + "technologies" + ].values() + for subcarrier in carrier + ] + if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] else [] ) @@ -773,7 +779,7 @@ def add_heating_capacities_installed_before_baseyear( countries=snakemake.config["countries"], capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"], lifetime_values=snakemake.params.costs["fill_values"], - tyndp_renewable_profiles=tyndp_renewable_profiles, + tyndp_renewable_carriers=tyndp_renewable_carriers, ) if options["heating"]: diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index a41474bd8d..65f5a8994b 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -90,6 +90,7 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: "offwind-ac": "Wind_Offshore", "offwind-dc": "Wind_Offshore", "offwind-float": "Wind_Offshore", + "offwind": "Wind_Offshore", "onwind": "Wind_Onshore", "solar": "LFSolarPV", "solar-hsat": "LFSolarPV", From f2af81bbdf0e9e2b7f4b39a801674be914d98c8e Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 12 Jun 2025 17:58:16 +0200 Subject: [PATCH 053/165] bugfix: include missing offshore node PECD data as empty columns and replace UK with GB naming convention --- scripts/clean_pecd_data.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 65f5a8994b..f7a2928f66 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -124,6 +124,13 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: with mp.Pool(processes=snakemake.threads) as pool: demand = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) - pecd_df = pd.concat(demand, axis=1) + pecd_df = ( + pd.concat(demand, axis=1) + .reindex(nodes, axis=1) + .fillna(0.0) # include missing node data with empty columns + .rename( + columns=lambda x: x.replace("UK", "GB") + ) # replace UK with GB for naming convention + ) pecd_df.to_csv(snakemake.output.pecd_data_clean) From 58f8040358b4fd127af1898ea5be4e83d5891195 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 13 Jun 2025 10:47:43 +0200 Subject: [PATCH 054/165] feat: improve myopic approach, removing previous assets --- config/test/config.tyndp.yaml | 1 + rules/solve_myopic.smk | 1 + scripts/add_brownfield.py | 27 ++++++++++++++++++++++++++- scripts/prepare_sector_network.py | 8 ++++++-- 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index bec2fcc420..92738a42fb 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -84,6 +84,7 @@ load: enable: false manual_adjustments: false supplement_synthetic: false + pypsa_eur: Bus: - AC diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 692785fa7c..9172a5eac5 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -91,6 +91,7 @@ rule add_brownfield: dynamic_ptes_capacity=config_provider( "sector", "district_heating", "ptes", "dynamic_capacity" ), + offshore_hubs=config_provider("sector", "offshore_hubs"), input: unpack(input_profile_tech_brownfield), unpack(input_profile_tech_brownfied_pecd), diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index cf4eb621bf..9de3c19ceb 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -33,6 +33,7 @@ def add_brownfield( h2_retrofit=False, h2_retrofit_capacity_per_ch4=None, capacity_threshold=None, + offshore_hubs=False, ): """ Add brownfield capacity from previous network. @@ -51,14 +52,34 @@ def add_brownfield( Ratio of hydrogen to methane capacity for pipeline retrofitting capacity_threshold : float Threshold for removing assets with low capacity + offshore_hubs : bool + Whether to enable offshore hubs """ logger.info(f"Preparing brownfield for the year {year}") # electric transmission grid set optimised capacities of previous as minimum n.lines.s_nom_min = n_p.lines.s_nom_opt - dc_i = n.links[n.links.carrier == "DC"].index + dc_i = n.links[(n.links.carrier == "DC") & (n.links.build_year < year)].index n.links.loc[dc_i, "p_nom_min"] = n_p.links.loc[dc_i, "p_nom_opt"] + # offshore generators and links (H2 pipeline, DC links, and electrolysers) capacities set optimised + # capacities of previous as minimum if the value exceeds the minimum value + if offshore_hubs: + off_li_i = n_p.links.index[n_p.links.index.str.contains("Offshore")] + off_gens_i = n_p.generators.index[n_p.generators.index.str.contains("offwind")] + off_i = {"Link": off_li_i, "Generator": off_gens_i, "Store": pd.Index([])} + n.links.loc[off_li_i, "p_nom_min"] = pd.concat( + [n.links.loc[off_li_i, "p_nom_min"], n_p.links.loc[off_li_i, "p_nom_opt"]], + axis=1, + ).max(axis=1) + n.generators.loc[off_gens_i, "p_nom_min"] = pd.concat( + [ + n.generators.loc[off_gens_i, "p_nom_min"], + n_p.generators.loc[off_gens_i, "p_nom_opt"], + ], + axis=1, + ).max(axis=1) + for c in n_p.iterate_components(["Link", "Generator", "Store"]): attr = "e" if c.name == "Store" else "p" @@ -89,6 +110,9 @@ def add_brownfield( chp_heat[c.df.loc[chp_heat, f"{attr}_nom_opt"] < threshold_chp_heat], ) + # remove offshore hubs assets as they are added for each planning horizon (Offshore Hubs methodology) + n_p.remove(c.name, off_i[c.name]) + n_p.remove( c.name, c.df.index[ @@ -370,6 +394,7 @@ def update_dynamic_ptes_capacity( h2_retrofit=snakemake.params.H2_retrofit, h2_retrofit_capacity_per_ch4=snakemake.params.H2_retrofit_capacity_per_CH4, capacity_threshold=snakemake.params.threshold_capacity, + offshore_hubs=snakemake.params.offshore_hubs, ) disable_grid_expansion_if_limit_hit(n) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index dc1d6e12a1..d76e0d6c35 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3135,6 +3135,7 @@ def add_offshore_generators_tyndp( marginal_cost=costs.at["offwind", "marginal_cost"], efficiency=costs.at["offwind", "efficiency"], p_max_pu=p_max_pu, + build_year=pyear, lifetime=costs.at["offwind", "lifetime"], ) @@ -3196,6 +3197,7 @@ def add_offshore_electrolysers_tyndp( carrier="H2 Electrolysis", efficiency=costs.at["electrolysis", "efficiency"], capital_cost=costs.at["electrolysis", "capital_cost"], + build_year=pyear, lifetime=costs.at["electrolysis", "lifetime"], ) @@ -3263,7 +3265,7 @@ def add_offshore_grid_tyndp( # Add DC grid connections offshore_grid_dc = offshore_grid.query("carrier=='DC'").copy() offshore_grid_dc.index = offshore_grid_dc.apply( - lambda x: f"{x.bus0}-{x.bus1}-DC", axis=1 + lambda x: f"{x.bus0}-{x.bus1}-Offshore DC", axis=1 ) offshore_grid_dc.loc[:, "capital_cost"] = ( annuity_factor.get("HVDC submarine") * offshore_grid_dc["capex"] @@ -3281,13 +3283,14 @@ def add_offshore_grid_tyndp( p_max_pu=offshore_grid_dc.p_max_pu, capital_cost=offshore_grid_dc.capital_cost, carrier="DC", + build_year=pyear, lifetime=costs.at["HVDC submarine", "lifetime"], ) # Add H2 pipeline connections offshore_grid_h2 = offshore_grid.query("carrier=='H2'").copy() offshore_grid_h2.index = offshore_grid_h2.apply( - make_index, axis=1, prefix="H2 pipeline" + make_index, axis=1, prefix="Offshore H2 pipeline" ) offshore_grid_h2.loc[:, "capital_cost"] = ( annuity_factor.get("H2 (g) submarine pipeline") * offshore_grid_h2["capex"] @@ -3306,6 +3309,7 @@ def add_offshore_grid_tyndp( p_max_pu=offshore_grid_h2.p_max_pu, capital_cost=offshore_grid_h2.capital_cost, carrier="H2 pipeline", + build_year=pyear, lifetime=costs.at["H2 (g) submarine pipeline", "lifetime"], ) From cf11da32476c428bef8f0d592f83472ee34a3f93 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 13 Jun 2025 11:28:40 +0200 Subject: [PATCH 055/165] bugfix: fix pylint and remove hydro from test config --- config/test/config.tyndp.yaml | 2 +- scripts/add_brownfield.py | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index a363a06c6a..38a12ae62a 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -56,7 +56,7 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh, hydro] + renewable_carriers: [solar, solar-hsat, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] atlite: default_cutout: europe-2013-03-sarah3-era5 diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 4c8512980a..127c24d0bd 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -206,7 +206,9 @@ def disable_grid_expansion_if_limit_hit(n): n.global_constraints.drop(name, inplace=True) -def adjust_renewable_profiles(n, input_profiles, params, year): +def adjust_renewable_profiles( + n, input_profiles, params, year, tyndp_renewable_carriers +): """ Adjusts renewable profiles according to the renewable technology specified, using the latest year below or equal to the selected year. @@ -219,17 +221,6 @@ def adjust_renewable_profiles(n, input_profiles, params, year): ) # TODO: hotfix remove filter for tyndp_renewable_carriers after tyndp generators are added - tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] - if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] - else [] - ) if len(tyndp_renewable_carriers) > 0: logger.info( f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." @@ -365,7 +356,21 @@ def update_dynamic_ptes_capacity( n = pypsa.Network(snakemake.input.network) - adjust_renewable_profiles(n, snakemake.input, snakemake.params, year) + tyndp_renewable_carriers = ( + [ + subcarrier + for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ + "technologies" + ].values() + for subcarrier in carrier + ] + if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] + else [] + ) + + adjust_renewable_profiles( + n, snakemake.input, snakemake.params, year, tyndp_renewable_carriers + ) add_build_year_to_new_assets(n, year) From d84159ffcbb2d58d8ae2f03c7304239d627f29a6 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 13 Jun 2025 11:30:39 +0200 Subject: [PATCH 056/165] bugfix: add existing res dependent on renewable carriers list --- rules/solve_myopic.smk | 1 + scripts/add_existing_baseyear.py | 91 ++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 41 deletions(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index ac721e8d2c..dce9c4e3ad 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -9,6 +9,7 @@ rule add_existing_baseyear: sector=config_provider("sector"), electricity=config_provider("electricity"), existing_capacities=config_provider("existing_capacities"), + carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), energy_totals_year=config_provider("energy", "energy_totals_year"), diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index d14ad9a850..8609e02581 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -68,6 +68,7 @@ def add_existing_renewables( costs: pd.DataFrame, df_agg: pd.DataFrame, countries: list[str], + renewable_carriers: list[str], tyndp_renewable_carriers: list[str], ) -> None: """ @@ -83,6 +84,8 @@ def add_existing_renewables( Network containing topology and generator data countries : list List of country codes to consider + renewable_carriers: list + List of renewable carriers in the network tyndp_renewable_carriers: list List of renewable technologies from TYNDP @@ -97,8 +100,7 @@ def add_existing_renewables( logger.info( f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." ) - for k in tyndp_renewable_carriers: - tech_map.pop(k, None) + renewable_carriers = set(renewable_carriers) - set(tyndp_renewable_carriers) irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() irena = irena.query("Country in @countries") @@ -107,45 +109,46 @@ def add_existing_renewables( irena = irena.unstack().reset_index() for carrier, tech in tech_map.items(): - df = ( - irena[irena.Technology.str.contains(tech)] - .drop(columns=["Technology"]) - .set_index("Country") - ) - df.columns = df.columns.astype(int) - - # calculate yearly differences - df.insert(loc=0, value=0.0, column="1999") - df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0) - - # distribute capacities among generators potential (p_nom_max) - gen_i = n.generators.query("carrier == @carrier").index - carrier_gens = n.generators.loc[gen_i] - res_capacities = [] - for country, group in carrier_gens.groupby( - carrier_gens.bus.map(n.buses.country) - ): - fraction = group.p_nom_max / group.p_nom_max.sum() - res_capacities.append(cartesian(df.loc[country], fraction)) - res_capacities = pd.concat(res_capacities, axis=1).T - - for year in res_capacities.columns: - for gen in res_capacities.index: - bus_bin = re.sub(f" {carrier}.*", "", gen) - bus, bin_id = bus_bin.rsplit(" ", maxsplit=1) - name = f"{bus_bin} {carrier}-{year}" - capacity = res_capacities.loc[gen, year] - if capacity > 0.0: - cost_key = carrier.split("-", maxsplit=1)[0] - df_agg.at[name, "Fueltype"] = carrier - df_agg.at[name, "Capacity"] = capacity - df_agg.at[name, "DateIn"] = year - df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"] - df_agg.at[name, "DateOut"] = ( - year + costs.at[cost_key, "lifetime"] - 1 - ) - df_agg.at[name, "bus"] = bus - df_agg.at[name, "resource_class"] = bin_id + if carrier in renewable_carriers: + df = ( + irena[irena.Technology.str.contains(tech)] + .drop(columns=["Technology"]) + .set_index("Country") + ) + df.columns = df.columns.astype(int) + + # calculate yearly differences + df.insert(loc=0, value=0.0, column="1999") + df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0) + + # distribute capacities among generators potential (p_nom_max) + gen_i = n.generators.query("carrier == @carrier").index + carrier_gens = n.generators.loc[gen_i] + res_capacities = [] + for country, group in carrier_gens.groupby( + carrier_gens.bus.map(n.buses.country) + ): + fraction = group.p_nom_max / group.p_nom_max.sum() + res_capacities.append(cartesian(df.loc[country], fraction)) + res_capacities = pd.concat(res_capacities, axis=1).T + + for year in res_capacities.columns: + for gen in res_capacities.index: + bus_bin = re.sub(f" {carrier}.*", "", gen) + bus, bin_id = bus_bin.rsplit(" ", maxsplit=1) + name = f"{bus_bin} {carrier}-{year}" + capacity = res_capacities.loc[gen, year] + if capacity > 0.0: + cost_key = carrier.split("-", maxsplit=1)[0] + df_agg.at[name, "Fueltype"] = carrier + df_agg.at[name, "Capacity"] = capacity + df_agg.at[name, "DateIn"] = year + df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"] + df_agg.at[name, "DateOut"] = ( + year + costs.at[cost_key, "lifetime"] - 1 + ) + df_agg.at[name, "bus"] = bus + df_agg.at[name, "resource_class"] = bin_id df_agg["resource_class"] = df_agg["resource_class"].fillna(0) @@ -159,6 +162,7 @@ def add_power_capacities_installed_before_baseyear( countries: list[str], capacity_threshold: float, lifetime_values: dict[str, float], + renewable_carriers: list[str], tyndp_renewable_carriers: list[str], ) -> None: """ @@ -182,6 +186,8 @@ def add_power_capacities_installed_before_baseyear( Minimum capacity threshold lifetime_values : dict Default values for missing data + renewable_carriers: list + List of renewable carriers in the network tyndp_renewable_carriers: list List of renewable technologies from TYNDP """ @@ -236,6 +242,7 @@ def add_power_capacities_installed_before_baseyear( costs=costs, n=n, countries=countries, + renewable_carriers=renewable_carriers, tyndp_renewable_carriers=tyndp_renewable_carriers, ) # drop assets which are already phased out / decommissioned @@ -741,6 +748,7 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector + renewable_carriers = snakemake.params.carriers tyndp_renewable_carriers = ( [ subcarrier @@ -779,6 +787,7 @@ def add_heating_capacities_installed_before_baseyear( countries=snakemake.config["countries"], capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"], lifetime_values=snakemake.params.costs["fill_values"], + renewable_carriers=renewable_carriers, tyndp_renewable_carriers=tyndp_renewable_carriers, ) From e0635cbb9f5337e47c8160ab71ab4ef1a5bd10d5 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 13 Jun 2025 11:32:12 +0200 Subject: [PATCH 057/165] feature: add 2050 pecd renewable profiles explicitly and filter for snapshots when cleaning data --- rules/build_electricity.smk | 1 + scripts/build_renewable_profiles_pecd.py | 3 ++- scripts/clean_pecd_data.py | 22 ++++++++++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index c36e9e774e..8513eb5694 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -397,6 +397,7 @@ rule clean_pecd_data: params: scenario=config_provider("tyndp_scenario"), snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 23cee8cd9a..c1793876cd 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -61,6 +61,7 @@ "Planning horizon doesn't match available TYNDP PECD data. " f"Falling back to previous available year {year}." ) + year_i = str(year) if year == 2050: logger.warning( "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." @@ -74,7 +75,7 @@ .rename_axis("time") .reset_index() .melt(id_vars=["time"], var_name="bus", value_name="profile") - .assign(bin=0, year=year) + .assign(bin=0, year=year_i) .set_index(["time", "bus", "bin", "year"]) .to_xarray() ) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index f7a2928f66..98f6826dec 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -36,7 +36,14 @@ logger = logging.getLogger(__name__) -def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: str): +def read_pecd_file( + node: list, + fn_pecd: str, + cyear: str, + pyear: str, + technology: str, + sns: pd.DatetimeIndex, +): fn = Path(fn_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") if not os.path.isfile(fn): @@ -45,8 +52,12 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: pecd_bus = pd.read_csv( fn, skiprows=10, - usecols=lambda name: name == "Date" or name == "Hour" or name == str(cyear), - ) + usecols=lambda name: name == "Date" + or name == "Hour" + or name == str(cyear) + or name == str(float(cyear)), + ).rename(columns={str(float(cyear)): str(cyear)}) + datetime_str = f"{cyear}." + pecd_bus["Date"].str.cat( (pecd_bus["Hour"] - 1).astype(str), sep=" " ) @@ -54,6 +65,7 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: pecd_bus.set_index(pd.to_datetime(datetime_str, format="%Y.%d.%m. %H")) .drop(columns=["Date", "Hour"]) .rename(columns={str(cyear): node}) + .loc[sns] # filter for snapshots only ) return cf_pecd @@ -73,7 +85,8 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: set_scenario_config(snakemake) # Climate year from snapshots - cyear = get_snapshots(snakemake.params.snapshots)[0].year + sns = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + cyear = sns[0].year if int(cyear) < 1982 or int(cyear) > 2019: # TODO: Note that because of this fallback, the snapshots of the profiles will not always match with the model snapshots logger.warning( @@ -119,6 +132,7 @@ def read_pecd_file(node: list, fn_pecd: str, cyear: str, pyear: str, technology: cyear=cyear, pyear=pyear, technology=pecd_tech, + sns=sns, ) with mp.Pool(processes=snakemake.threads) as pool: From 11422163dbd89fdfde6f9c1854a17f7f1a7e6a6f Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 13 Jun 2025 12:15:31 +0200 Subject: [PATCH 058/165] feat: introduce a constraint to limit expansion of collocated technologies that use the same potential --- rules/solve_myopic.smk | 5 +++ scripts/prepare_sector_network.py | 2 +- scripts/solve_network.py | 59 +++++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 9172a5eac5..12ec994491 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -140,6 +140,11 @@ rule solve_sector_network_myopic: "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc" ), costs=resources("costs_{planning_horizons}.csv"), + offshore_zone_trajectories=lambda w: ( + resources("offshore_zone_trajectories.csv") + if config_provider("sector", "offshore_hubs")(w) + else [] + ), output: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index d76e0d6c35..a26b3f7aa0 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3129,7 +3129,7 @@ def add_offshore_generators_tyndp( carrier=offshore_generators.carrier, p_nom=offshore_generators.p_nom_min, p_nom_min=offshore_generators.p_nom_min, - p_nom_max=offshore_generators.p_nom_min, + p_nom_max=offshore_generators.p_nom_max, p_nom_extendable=offshore_generators.p_nom_extendable, capital_cost=offshore_generators.capital_cost, marginal_cost=costs.at["offwind", "marginal_cost"], diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ee305b1800..b4a73cb8e0 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1112,6 +1112,45 @@ def add_import_limit_constraint(n: pypsa.Network, sns: pd.DatetimeIndex): n.model.add_constraints(lhs, limit_sense, rhs, name="import_limit") +def add_offshore_hubs_constraint( + n, planning_horizons: int | None, offshore_zone_trajectories_fn +): + """ + Add two constraints on offshore hubs. + + 1. Constraint expansion of DC and H2 sitting on the same location, as the sum of the two capacities cannot exceed the layer potential. + 2. Constraint the maximum potential per zone. + + Parameters + ---------- + n : pypsa.Network + The PyPSA network instance + planning_horizons : int, optional + The current planning horizon year or None in perfect foresight + offshore_zone_trajectories_fn: str + Path to the dataFrame containing the offshore zone potentials trajectories + """ + # Constraint DC / H2 expansion on the same layer + h2_gens = n.generators.loc[n.generators.carrier.str.contains("h2")] + h2_gens_i = h2_gens.index + dc_gens_i = h2_gens_i.str.replace("h2", "dc").str.replace(" H2", "") + + off_electrolysers = n.links.loc[ + n.links.index.str.contains("Offshore Electrolysis") + ].set_index("bus1") + eff = ( + off_electrolysers.loc[h2_gens.bus] + .set_index(h2_gens_i) + .efficiency.rename_axis("Generator-ext") + ) + p_nom = n.model["Generator-p_nom"] + lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff + + rhs = n.generators.loc[dc_gens_i].p_nom_max.rename_axis("Generator-ext") + + n.model.add_constraints(lhs <= rhs, name="Generator-off_h2_dc_pot") + + def add_co2_atmosphere_constraint(n, snapshots): glcs = n.global_constraints[n.global_constraints.type == "co2_atmosphere"] @@ -1136,7 +1175,10 @@ def add_co2_atmosphere_constraint(n, snapshots): def extra_functionality( - n: pypsa.Network, snapshots: pd.DatetimeIndex, planning_horizons: str | None = None + n: pypsa.Network, + snapshots: pd.DatetimeIndex, + planning_horizons: str | None = None, + offshore_zone_trajectories_fn: str | None = None, ) -> None: """ Add custom constraints and functionality. @@ -1149,6 +1191,8 @@ def extra_functionality( Simulation timesteps planning_horizons : str, optional The current planning horizon year or None in perfect foresight + offshore_zone_trajectories_fn: str, optional + Path to the dataFrame containing the offshore zone potentials trajectories Collects supplementary constraints which will be passed to ``pypsa.optimization.optimize``. @@ -1205,6 +1249,11 @@ def extra_functionality( if config["sector"]["imports"]["enable"]: add_import_limit_constraint(n, snapshots) + if config["sector"]["offshore_hubs"]: + add_offshore_hubs_constraint( + n, int(planning_horizons), offshore_zone_trajectories_fn + ) + if n.params.custom_extra_functionality: source_path = n.params.custom_extra_functionality assert os.path.exists(source_path), f"{source_path} does not exist" @@ -1250,6 +1299,7 @@ def solve_network( solving: dict, rule_name: str | None = None, planning_horizons: str | None = None, + offshore_zone_trajectories_fn: str | None = None, **kwargs, ) -> None: """ @@ -1269,6 +1319,8 @@ def solve_network( Name of the snakemake rule being executed planning_horizons : str, optional The current planning horizon year or None in perfect foresight + offshore_zone_trajectories_fn : str, optional + Path to dataFrame containing the offshore zone potentials trajectories **kwargs Additional keyword arguments passed to the solver @@ -1297,7 +1349,9 @@ def solve_network( ) kwargs["solver_name"] = solving["solver"]["name"] kwargs["extra_functionality"] = partial( - extra_functionality, planning_horizons=planning_horizons + extra_functionality, + planning_horizons=planning_horizons, + offshore_zone_trajectories_fn=offshore_zone_trajectories_fn, ) kwargs["transmission_losses"] = cf_solving.get("transmission_losses", False) kwargs["linearized_unit_commitment"] = cf_solving.get( @@ -1403,6 +1457,7 @@ def solve_network( planning_horizons=planning_horizons, rule_name=snakemake.rule, log_fn=snakemake.log.solver, + offshore_zone_trajectories_fn=snakemake.input.offshore_zone_trajectories, ) logger.info(f"Maximum memory usage: {mem.mem_usage}") From 7b297a36adba7a480c4656e7882ca9cfe589ff89 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 13 Jun 2025 13:08:00 +0200 Subject: [PATCH 059/165] bugfix: fix params for add_existing_baseyear solve_perfect --- rules/solve_perfect.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 3660dbf152..545b547705 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -7,6 +7,7 @@ rule add_existing_baseyear: sector=config_provider("sector"), electricity=config_provider("electricity"), existing_capacities=config_provider("existing_capacities"), + carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), energy_totals_year=config_provider("energy", "energy_totals_year"), From 7512f30591eddb23112a84c2b9a3c9b08645c265 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Sun, 15 Jun 2025 21:52:21 +0200 Subject: [PATCH 060/165] feat: add zone potential constraint --- scripts/solve_network.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index b4a73cb8e0..3f33904f6e 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1138,18 +1138,31 @@ def add_offshore_hubs_constraint( off_electrolysers = n.links.loc[ n.links.index.str.contains("Offshore Electrolysis") ].set_index("bus1") - eff = ( - off_electrolysers.loc[h2_gens.bus] - .set_index(h2_gens_i) - .efficiency.rename_axis("Generator-ext") - ) - p_nom = n.model["Generator-p_nom"] + eff = off_electrolysers.loc[h2_gens.bus].set_index(h2_gens_i).efficiency + p_nom = n.model["Generator-p_nom"].rename({"Generator-ext": "Generator"}) lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff - rhs = n.generators.loc[dc_gens_i].p_nom_max.rename_axis("Generator-ext") + rhs = n.generators.loc[dc_gens_i].p_nom_max n.model.add_constraints(lhs <= rhs, name="Generator-off_h2_dc_pot") + # Constraint the maximum potential per zone + limit = ( + pd.read_csv(offshore_zone_trajectories_fn, index_col=0) + .query("pyear == @planning_horizons") + .p_nom_max + ) + + off_gens_i = n.generators.loc[n.generators.index.str.contains("offwind")].index + eff = eff.reindex(off_gens_i, fill_value=1) + grouper = n.generators.loc[off_gens_i].bus.map(n.buses.location) + idx = pd.Index(set(limit.index).intersection(grouper)) + + lhs = (p_nom.loc[off_gens_i] / eff).groupby(grouper).sum().loc[idx] + rhs = limit.loc[idx] + + n.model.add_constraints(lhs <= rhs, name="Generator-off_zone_pot") + def add_co2_atmosphere_constraint(n, snapshots): glcs = n.global_constraints[n.global_constraints.type == "co2_atmosphere"] From fe8cb522037efa0e083331a17a51e8d40ff66cac Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 16 Jun 2025 11:55:42 +0200 Subject: [PATCH 061/165] bugfix: change year in renewable profile pecd back to integer --- scripts/build_renewable_profiles_pecd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index c1793876cd..68fa9acf85 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -61,7 +61,7 @@ "Planning horizon doesn't match available TYNDP PECD data. " f"Falling back to previous available year {year}." ) - year_i = str(year) + year_i = year if year == 2050: logger.warning( "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." From 96aef7839211f2e632a52fbbce053e91dc81a2d2 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 16 Jun 2025 13:03:16 +0200 Subject: [PATCH 062/165] fix: address patches used to develop parallel features --- scripts/add_brownfield.py | 27 +++------------------------ scripts/add_existing_baseyear.py | 25 ------------------------- scripts/prepare_sector_network.py | 9 ++------- 3 files changed, 5 insertions(+), 56 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 48a0ffe889..1c36061880 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -230,9 +230,7 @@ def disable_grid_expansion_if_limit_hit(n): n.global_constraints.drop(name, inplace=True) -def adjust_renewable_profiles( - n, input_profiles, params, year, tyndp_renewable_carriers -): +def adjust_renewable_profiles(n, input_profiles, params, year): """ Adjusts renewable profiles according to the renewable technology specified, using the latest year below or equal to the selected year. @@ -244,12 +242,7 @@ def adjust_renewable_profiles( pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill() ) - # TODO: hotfix remove filter for tyndp_renewable_carriers after tyndp generators are added - if len(tyndp_renewable_carriers) > 0: - logger.info( - f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." - ) - for carrier in set(params["carriers"]) - set(tyndp_renewable_carriers): + for carrier in set(params["carriers"]): if carrier == "hydro": continue @@ -380,21 +373,7 @@ def update_dynamic_ptes_capacity( n = pypsa.Network(snakemake.input.network) - tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] - if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] - else [] - ) - - adjust_renewable_profiles( - n, snakemake.input, snakemake.params, year, tyndp_renewable_carriers - ) + adjust_renewable_profiles(n, snakemake.input, snakemake.params, year) add_build_year_to_new_assets(n, year) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 8609e02581..be278d6843 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -69,7 +69,6 @@ def add_existing_renewables( df_agg: pd.DataFrame, countries: list[str], renewable_carriers: list[str], - tyndp_renewable_carriers: list[str], ) -> None: """ Add existing renewable capacities to conventional power plant data. @@ -86,8 +85,6 @@ def add_existing_renewables( List of country codes to consider renewable_carriers: list List of renewable carriers in the network - tyndp_renewable_carriers: list - List of renewable technologies from TYNDP Returns ------- @@ -95,12 +92,6 @@ def add_existing_renewables( Modifies df_agg in-place """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} - # TODO: remove when TYNDP renewable generators are added - if len(tyndp_renewable_carriers) > 0: - logger.info( - f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." - ) - renewable_carriers = set(renewable_carriers) - set(tyndp_renewable_carriers) irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() irena = irena.query("Country in @countries") @@ -163,7 +154,6 @@ def add_power_capacities_installed_before_baseyear( capacity_threshold: float, lifetime_values: dict[str, float], renewable_carriers: list[str], - tyndp_renewable_carriers: list[str], ) -> None: """ Add power generation capacities installed before base year. @@ -188,8 +178,6 @@ def add_power_capacities_installed_before_baseyear( Default values for missing data renewable_carriers: list List of renewable carriers in the network - tyndp_renewable_carriers: list - List of renewable technologies from TYNDP """ logger.debug(f"Adding power capacities installed before {baseyear}") @@ -243,7 +231,6 @@ def add_power_capacities_installed_before_baseyear( n=n, countries=countries, renewable_carriers=renewable_carriers, - tyndp_renewable_carriers=tyndp_renewable_carriers, ) # drop assets which are already phased out / decommissioned phased_out = df_agg[df_agg["DateOut"] < baseyear].index @@ -749,17 +736,6 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector renewable_carriers = snakemake.params.carriers - tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] - if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] - else [] - ) baseyear = snakemake.params.baseyear @@ -788,7 +764,6 @@ def add_heating_capacities_installed_before_baseyear( capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"], lifetime_values=snakemake.params.costs["fill_values"], renewable_carriers=renewable_carriers, - tyndp_renewable_carriers=tyndp_renewable_carriers, ) if options["heating"]: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index a26b3f7aa0..0c9aeef545 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3108,18 +3108,13 @@ def add_offshore_generators_tyndp( techs = ["H2 " + tech_i if "h2" in tech_i else tech_i for tech_i in techs] with xr.open_dataset(fn) as ds: - ds = ds.sel( - year=pyear, bin=0, time=n.snapshots, drop=True - ) # ToDo Remove time sel once sns are filtered in PECD data + ds = ds.sel(year=pyear, bin=0, time=n.snapshots, drop=True) p_max_pu_i = ds["profile"].to_pandas() for tech_i in techs: p_max_pu.append(p_max_pu_i.rename(columns=lambda x: x + " " + tech_i)) - p_max_pu = pd.concat(p_max_pu, axis=1) - p_max_pu = p_max_pu.reindex( - offshore_generators.index, axis=1, fill_value=0 - ) # ToDo Simplify once missing nodes are addressed in PECD data + p_max_pu = pd.concat(p_max_pu, axis=1)[offshore_generators.index] # Add generators to the network n.add( From ef313b36545abd292e56a593072dded8d63e4b86 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 16 Jun 2025 13:25:41 +0200 Subject: [PATCH 063/165] fix: avoid adding an empty constraint --- scripts/solve_network.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 3f33904f6e..161275112b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1140,11 +1140,12 @@ def add_offshore_hubs_constraint( ].set_index("bus1") eff = off_electrolysers.loc[h2_gens.bus].set_index(h2_gens_i).efficiency p_nom = n.model["Generator-p_nom"].rename({"Generator-ext": "Generator"}) - lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff + lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff rhs = n.generators.loc[dc_gens_i].p_nom_max - n.model.add_constraints(lhs <= rhs, name="Generator-off_h2_dc_pot") + if not lhs.empty: + n.model.add_constraints(lhs <= rhs, name="Generator-off_h2_dc_pot") # Constraint the maximum potential per zone limit = ( From 1ff4a9438a9ad706a8b9eabb6e72a6365f9faed6 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 16 Jun 2025 13:29:40 +0200 Subject: [PATCH 064/165] doc: adjust license identifier --- scripts/build_tyndp_offshore_hubs.py | 2 +- scripts/solve_network.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index d7b28cfb06..117de81e4f 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: : 2024 The PyPSA-Eur Authors +# SPDX-FileCopyrightText: : Open Energy Transition gGmbH # # SPDX-License-Identifier: MIT """ diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 161275112b..bdae17468b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH and contributors to PyPSA-Eur # # SPDX-License-Identifier: MIT """ From 078afaf7640e5718d9daa2c040ebf106377a262c Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 16 Jun 2025 14:53:09 +0200 Subject: [PATCH 065/165] fix: address pecd profiles mapping in add_brownfield --- rules/solve_myopic.smk | 1 + scripts/add_brownfield.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index ebd3d3e8dc..772bd56cc0 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -87,6 +87,7 @@ rule add_brownfield: electricity=config_provider("electricity"), drop_leap_day=config_provider("enable", "drop_leap_day"), carriers=config_provider("electricity", "renewable_carriers"), + carriers_pecd=config_provider("electricity", "pecd_renewable_profiles"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), tes=config_provider("sector", "tes"), dynamic_ptes_capacity=config_provider( diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1c36061880..dd567d4f22 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -242,11 +242,23 @@ def adjust_renewable_profiles(n, input_profiles, params, year): pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill() ) + fn_map = {i: i for i in params["carriers"]} + if params["carriers_pecd"].get("enable", False): + fn_map.update( + { + vi: k + for k, v in params["carriers_pecd"]["technologies"].items() + for vi in v + } + ) + for carrier in set(params["carriers"]): if carrier == "hydro": continue - with xr.open_dataset(getattr(input_profiles, "profile_" + carrier)) as ds: + with xr.open_dataset( + getattr(input_profiles, "profile_" + fn_map[carrier]) + ) as ds: if ds.indexes["bus"].empty or "year" not in ds.indexes: continue From 1f540fd928165268e3c0482278ac2490c156b664 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 16 Jun 2025 17:20:34 +0200 Subject: [PATCH 066/165] refactor: refactor myopic approach for offshore assets --- scripts/add_brownfield.py | 53 +++++++++++++++++++------------ scripts/prepare_sector_network.py | 4 --- scripts/solve_network.py | 16 +++++++--- 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index dd567d4f22..10cef2bbc7 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -62,24 +62,6 @@ def add_brownfield( dc_i = n.links[(n.links.carrier == "DC") & (n.links.build_year < year)].index n.links.loc[dc_i, "p_nom_min"] = n_p.links.loc[dc_i, "p_nom_opt"] - # offshore generators and links (H2 pipeline, DC links, and electrolysers) capacities set optimised - # capacities of previous as minimum if the value exceeds the minimum value - if offshore_hubs: - off_li_i = n_p.links.index[n_p.links.index.str.contains("Offshore")] - off_gens_i = n_p.generators.index[n_p.generators.index.str.contains("offwind")] - off_i = {"Link": off_li_i, "Generator": off_gens_i, "Store": pd.Index([])} - n.links.loc[off_li_i, "p_nom_min"] = pd.concat( - [n.links.loc[off_li_i, "p_nom_min"], n_p.links.loc[off_li_i, "p_nom_opt"]], - axis=1, - ).max(axis=1) - n.generators.loc[off_gens_i, "p_nom_min"] = pd.concat( - [ - n.generators.loc[off_gens_i, "p_nom_min"], - n_p.generators.loc[off_gens_i, "p_nom_opt"], - ], - axis=1, - ).max(axis=1) - for c in n_p.iterate_components(["Link", "Generator", "Store"]): attr = "e" if c.name == "Store" else "p" @@ -110,9 +92,6 @@ def add_brownfield( chp_heat[c.df.loc[chp_heat, f"{attr}_nom_opt"] < threshold_chp_heat], ) - # remove offshore hubs assets as they are added for each planning horizon (Offshore Hubs methodology) - n_p.remove(c.name, off_i[c.name]) - n_p.remove( c.name, c.df.index[ @@ -134,6 +113,38 @@ def add_brownfield( for tattr in n.component_attrs[c.name].index[selection]: n.import_series_from_dataframe(c.pnl[tattr], c.name, tattr) + # adjust TYNDP offshore expansion by subtracting existing capacity from previous years from current year total capacity and potential + if offshore_hubs: + filter = {"Link": "Offshore", "Generator": "offwind"} + for c in n.iterate_components(["Link", "Generator"]): + off_fixed_i = c.df[ + (c.df.index.str.contains(filter[c.name])) & (c.df.build_year != year) + ].index + off_i = c.df[ + (c.df.index.str.contains(filter[c.name])) & (c.df.build_year == year) + ].index + + off_capacity = c.df.loc[off_i, "p_nom"] + off_potential = c.df.loc[off_i, "p_nom_max"] + already_existing = ( + c.df.loc[off_fixed_i, "p_nom_opt"] + .rename(lambda x: x.split("-2")[0] + f"-{year}") + .groupby(level=0) + .sum() + ) + remaining_capacity = ( + off_capacity + - already_existing.reindex(index=off_capacity.index).fillna(0) + ).clip(lower=0) + remaining_potential = ( + off_potential + - already_existing.reindex(index=off_capacity.index).fillna(0) + ).clip( + lower=0 + ) # this should anyway never be negative. We will still clip to account for rounding errors + c.df.loc[off_i, ["p_nom_min", "p_nom"]] = remaining_capacity + c.df.loc[off_i, "p_nom_max"] = remaining_potential + # deal with gas network if h2_retrofit: # subtract the already retrofitted from the maximum capacity diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 0c9aeef545..46b4ed0930 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3130,7 +3130,6 @@ def add_offshore_generators_tyndp( marginal_cost=costs.at["offwind", "marginal_cost"], efficiency=costs.at["offwind", "efficiency"], p_max_pu=p_max_pu, - build_year=pyear, lifetime=costs.at["offwind", "lifetime"], ) @@ -3192,7 +3191,6 @@ def add_offshore_electrolysers_tyndp( carrier="H2 Electrolysis", efficiency=costs.at["electrolysis", "efficiency"], capital_cost=costs.at["electrolysis", "capital_cost"], - build_year=pyear, lifetime=costs.at["electrolysis", "lifetime"], ) @@ -3278,7 +3276,6 @@ def add_offshore_grid_tyndp( p_max_pu=offshore_grid_dc.p_max_pu, capital_cost=offshore_grid_dc.capital_cost, carrier="DC", - build_year=pyear, lifetime=costs.at["HVDC submarine", "lifetime"], ) @@ -3304,7 +3301,6 @@ def add_offshore_grid_tyndp( p_max_pu=offshore_grid_h2.p_max_pu, capital_cost=offshore_grid_h2.capital_cost, carrier="H2 pipeline", - build_year=pyear, lifetime=costs.at["H2 (g) submarine pipeline", "lifetime"], ) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index bdae17468b..babc140e10 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1154,13 +1154,19 @@ def add_offshore_hubs_constraint( .p_nom_max ) - off_gens_i = n.generators.loc[n.generators.index.str.contains("offwind")].index + ext_i = n.generators.p_nom_extendable + off_i = n.generators.index.str.contains("offwind") + + off_gens_i = n.generators.loc[(off_i) & (ext_i)].index + grouper_ext = n.generators.loc[off_gens_i].bus.map(n.buses.location) + idx = pd.Index(set(limit.index).intersection(grouper_ext)) eff = eff.reindex(off_gens_i, fill_value=1) - grouper = n.generators.loc[off_gens_i].bus.map(n.buses.location) - idx = pd.Index(set(limit.index).intersection(grouper)) + lhs = (p_nom.loc[off_gens_i] / eff).groupby(grouper_ext).sum().loc[idx] - lhs = (p_nom.loc[off_gens_i] / eff).groupby(grouper).sum().loc[idx] - rhs = limit.loc[idx] + existing = n.generators.loc[(off_i) & ~(ext_i), "p_nom"] + grouper = n.generators.loc[existing.index].bus.map(n.buses.location) + existing = existing.groupby(grouper).sum().reindex(idx, fill_value=0) + rhs = limit.loc[idx] - existing n.model.add_constraints(lhs <= rhs, name="Generator-off_zone_pot") From 8525b1212f1d7cb36491c8146dbc2a934d5b52e7 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 17 Jun 2025 10:45:56 +0200 Subject: [PATCH 067/165] refactor: adjust nomenclature of offshore grid capacity --- scripts/build_tyndp_offshore_hubs.py | 2 +- scripts/prepare_sector_network.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 117de81e4f..643a81d324 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -115,7 +115,7 @@ def load_offshore_grid( "YEAR": "pyear", "SCENARIO": "scenario", "MARKET": "carrier", - "CAPACITY": "p_nom", + "CAPACITY": "p_nom_min", "CAPEX": "capex", "OPEX": "opex", } diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 46b4ed0930..0f357d530d 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3271,7 +3271,8 @@ def add_offshore_grid_tyndp( bus0=offshore_grid_dc.bus0, bus1=offshore_grid_dc.bus1, p_nom_extendable=offshore_grid_dc.p_nom_extendable, - p_nom=offshore_grid_dc.p_nom, + p_nom=offshore_grid_dc.p_nom_min, + p_nom_min=offshore_grid_dc.p_nom_min, p_min_pu=offshore_grid_dc.p_min_pu, p_max_pu=offshore_grid_dc.p_max_pu, capital_cost=offshore_grid_dc.capital_cost, From adaeb4c911d0b780b995f0ab5c74a5ffffb68888 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 17 Jun 2025 14:10:10 +0200 Subject: [PATCH 068/165] refactor: filter radial nodes out of buses --- scripts/build_tyndp_offshore_hubs.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 643a81d324..6f8ad2776c 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -45,14 +45,15 @@ def load_offshore_hubs(fn: str, countries: list[str]): "LON": "x", } - nodes = pd.read_excel( - fn, - sheet_name="NODE", - ).rename(columns=column_dict) - - mask = nodes["Bus"].str.contains("OH") - nodes.loc[mask, "location"] = nodes.loc[mask, "Bus"] - nodes.loc[:, "country"] = nodes.loc[:, "location"].str[:2] + nodes = ( + pd.read_excel(fn, sheet_name="NODE") + .rename(columns=column_dict) + .query("Bus.str.contains('OH')") + .assign( + location=lambda x: x.Bus, + country=lambda x: x.location.str[:2], + ) + ) nodes["geometry"] = nodes.apply(lambda row: Point(row["x"], row["y"]), axis=1) nodes = gpd.GeoDataFrame(nodes, geometry="geometry", crs=GEO_CRS) From ac94dfac8ac83f5b51d44bd6248e6dcaf2de2e7e Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 17 Jun 2025 14:11:16 +0200 Subject: [PATCH 069/165] refactor: assign radially connected electrolysers to home market nodes --- scripts/build_tyndp_offshore_hubs.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 6f8ad2776c..53191dff98 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -206,8 +206,8 @@ def load_offshore_electrolysers( DataFrame containing the formatted offshore electrolyser data. """ column_dict = { - "NODE": "location", - "OFFSHORE_NODE": "bus0", + "NODE": "bus0", + "OFFSHORE_NODE": "location", "OFFSHORE_NODE_TYPE": "type", "YEAR": "pyear", "SCENARIO": "scenario", @@ -231,9 +231,12 @@ def load_offshore_electrolysers( .query("pyear in @planning_horizons") .replace({"scenario": scenario_dict}) .query("scenario == @scenario") - .assign(bus1=lambda x: x.bus0 + " H2") + .assign(country=lambda x: x.bus0.str[:2], bus1=lambda x: x.bus0 + " H2") ) + mask = electrolysers["type"] == "Radial" + electrolysers.loc[mask, "bus1"] = electrolysers.loc[mask, "country"] + " H2 Z2" + electrolysers[["capex", "opex"]] = electrolysers[["capex", "opex"]].mul( 1e3 ) # kEUR/MW to EUR/MW @@ -244,9 +247,7 @@ def load_offshore_electrolysers( ].replace("UK", "GB", regex=True) # filter selected countries - electrolysers = electrolysers.assign(country=lambda x: x.bus0.str[:2]).query( - "country in @countries" - ) + electrolysers = electrolysers.query("country in @countries") return electrolysers From 396b3634173cfa5162c7648cbcc32841c81f8ffd Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 17 Jun 2025 14:11:48 +0200 Subject: [PATCH 070/165] fix: use tyndp data for electrolysers capital cost --- scripts/prepare_sector_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 0f357d530d..6db1ea5010 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3190,7 +3190,7 @@ def add_offshore_electrolysers_tyndp( p_nom_extendable=True, carrier="H2 Electrolysis", efficiency=costs.at["electrolysis", "efficiency"], - capital_cost=costs.at["electrolysis", "capital_cost"], + capital_cost=offshore_electrolysers.capital_cost, lifetime=costs.at["electrolysis", "lifetime"], ) From 97e9970859f5850cfb9440219cbb27881fff8726 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 18 Jun 2025 11:07:56 +0200 Subject: [PATCH 071/165] refactor: improve radial nodes exclusions --- scripts/build_tyndp_offshore_hubs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 53191dff98..844a9b6334 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -21,7 +21,7 @@ def load_offshore_hubs(fn: str, countries: list[str]): """ Load offshore hubs coordinates and format data. - Offshore Hubs (OH) nodes are situated offshore, while Offshore Radial (OR) nodes are located in the homeland market node. + Offshore Hubs (OH) nodes are situated offshore, while Offshore Radial (OR) nodes are removed from the data. Parameters ---------- @@ -40,7 +40,6 @@ def load_offshore_hubs(fn: str, countries: list[str]): column_dict = { "OFFSHORE_NODE": "Bus", "OFFSHORE_NODE_TYPE": "type", - "HOME_NODE": "location", "LAT": "y", "LON": "x", } @@ -48,11 +47,12 @@ def load_offshore_hubs(fn: str, countries: list[str]): nodes = ( pd.read_excel(fn, sheet_name="NODE") .rename(columns=column_dict) - .query("Bus.str.contains('OH')") + .query("type != 'Radial'") .assign( location=lambda x: x.Bus, country=lambda x: x.location.str[:2], ) + .drop(columns="HOME_NODE") ) nodes["geometry"] = nodes.apply(lambda row: Point(row["x"], row["y"]), axis=1) From 42bf3e3b64e39f316c76456f2e69f7356d4a9bf9 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 18 Jun 2025 11:22:44 +0200 Subject: [PATCH 072/165] fix: prevent filtering of grid data from expansion candidates --- scripts/build_tyndp_offshore_hubs.py | 32 +++++++++++++--------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 844a9b6334..cde98b6819 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -94,8 +94,7 @@ def load_offshore_grid( fn : str Path to the Excel file containing offshore grid data. nodes : pd.DataFrame - DataFrame containing node information (currently not used in function body - but may be needed for validation or future functionality). + DataFrame containing node information. scenario : str Scenario identifier to filter the grid data. Must be one of the scenario codes: "DE" (Distributed Energy), "GA" (Global Ambition), or @@ -128,17 +127,10 @@ def load_offshore_grid( } # Load reference grid - grid = ( - pd.read_excel( - fn, - sheet_name="Reference grid", - ) - .rename(columns=column_dict) - .assign( - p_min_pu=0, - p_max_pu=1, - ) - ) + grid = pd.read_excel( + fn, + sheet_name="Reference grid", + ).rename(columns=column_dict) grid["carrier"] = grid["carrier"].replace("E", "DC") grid = expand_all_scenario(grid, scenario_dict.values()).query( "scenario == @scenario" @@ -151,8 +143,8 @@ def load_offshore_grid( sheet_name="COST", ) .rename(columns=column_dict) - .query("pyear in @planning_horizons") .replace({"scenario": scenario_dict}) + .query("pyear in @planning_horizons and scenario == @scenario") ) grid_costs[["capex", "opex"]] = grid_costs[["capex", "opex"]].mul( 1e3 @@ -161,22 +153,28 @@ def load_offshore_grid( # Merge information grid = grid.merge( - grid_costs, how="left", on=["bus0", "bus1", "pyear", "scenario", "carrier"] + grid_costs, how="outer", on=["bus0", "bus1", "pyear", "scenario", "carrier"] + ).assign( + p_min_pu=0, + p_max_pu=1, ) # Assume non-extendable when missing data # TODO Validate assumption grid["p_nom_extendable"] = ~grid[["capex", "opex"]].isna().any(axis=1) grid[["capex", "opex"]] = grid[["capex", "opex"]].fillna(0) + grid["p_nom_min"] = grid["p_nom_min"].fillna(0) # Rename UK in GB grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB", regex=True) - # Filter selected countries + # Filter selected countries and nodes grid = grid.assign( country0=lambda x: x.bus0.str[:2], country1=lambda x: x.bus1.str[:2], - ).query("country0 in @countries and country1 in @countries") + ).query( + "country0 in @countries and country1 in @countries and ~bus0.str.contains('OR') and ~bus1.str.contains('OR')" + ) return grid From f552bd7023ce5d0d99dcbed771cacce8307d218d Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 18 Jun 2025 11:35:35 +0200 Subject: [PATCH 073/165] refactor: remove location from electrolysers --- scripts/build_tyndp_offshore_hubs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index cde98b6819..1c0203f439 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -205,7 +205,6 @@ def load_offshore_electrolysers( """ column_dict = { "NODE": "bus0", - "OFFSHORE_NODE": "location", "OFFSHORE_NODE_TYPE": "type", "YEAR": "pyear", "SCENARIO": "scenario", @@ -230,6 +229,7 @@ def load_offshore_electrolysers( .replace({"scenario": scenario_dict}) .query("scenario == @scenario") .assign(country=lambda x: x.bus0.str[:2], bus1=lambda x: x.bus0 + " H2") + .drop(columns="OFFSHORE_NODE") ) mask = electrolysers["type"] == "Radial" @@ -240,9 +240,9 @@ def load_offshore_electrolysers( ) # kEUR/MW to EUR/MW # rename UK in GB - electrolysers[["bus0", "bus1", "location"]] = electrolysers[ - ["bus0", "bus1", "location"] - ].replace("UK", "GB", regex=True) + electrolysers[["bus0", "bus1"]] = electrolysers[["bus0", "bus1"]].replace( + "UK", "GB", regex=True + ) # filter selected countries electrolysers = electrolysers.query("country in @countries") From ae82b6b32fcc84d44e9a131694c183eee2566225 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 18 Jun 2025 14:36:58 +0200 Subject: [PATCH 074/165] refactor: change approach for nodes and locations in generators --- scripts/build_tyndp_offshore_hubs.py | 185 +++++++++++++++------------ scripts/prepare_sector_network.py | 10 +- 2 files changed, 112 insertions(+), 83 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 1c0203f439..8191444146 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -19,7 +19,7 @@ def load_offshore_hubs(fn: str, countries: list[str]): """ - Load offshore hubs coordinates and format data. + Load and process offshore hub coordinates from Excel file. Offshore Hubs (OH) nodes are situated offshore, while Offshore Radial (OR) nodes are removed from the data. @@ -47,25 +47,15 @@ def load_offshore_hubs(fn: str, countries: list[str]): nodes = ( pd.read_excel(fn, sheet_name="NODE") .rename(columns=column_dict) - .query("type != 'Radial'") .assign( location=lambda x: x.Bus, country=lambda x: x.location.str[:2], ) - .drop(columns="HOME_NODE") ) nodes["geometry"] = nodes.apply(lambda row: Point(row["x"], row["y"]), axis=1) nodes = gpd.GeoDataFrame(nodes, geometry="geometry", crs=GEO_CRS) - # rename UK in GB - nodes[["Bus", "location", "country"]] = nodes[ - ["Bus", "location", "country"] - ].replace("UK", "GB", regex=True) - - # filter selected countries - nodes = nodes.query("country in @countries") - return nodes @@ -250,29 +240,36 @@ def load_offshore_electrolysers( return electrolysers -def get_shares(df, values, dropna=True): - df_share = ( - df.pivot_table( - index=["bus0", "type", "pyear", "scenario"], - values=values, - columns="carrier", - ) - .pipe(lambda df: df.div(df.sum(axis=1), axis=0)) - .melt(ignore_index=False, value_name="tech_share") - .reset_index() - ) - if dropna: - df_share = df_share.dropna(subset="tech_share") - return df_share +def collect_from_layer(generators_e, generators_l, nodes): + """ + Combine existing capacities with potentials and resolve bus allocations. + This function merges generator data from two sources: existing capacities (EXISTING sheet) + and potential capacities (LAYER sheet). It handles reallocation of radial wind farms by + correcting inconsistencies between EXISTING and LAYER data, particularly for offshore + radial connections that need to be mapped to hub connections. -def collect_from_layer(generators_e, generators_l): + Parameters + ---------- + generators_e : pd.DataFrame + Existing generator capacities. + generators_l : pd.DataFrame + Layer potential capacities. Contains candidate generators without explicit bus assignments. + nodes : pd.DataFrame + Node definitions. Used to deduce bus assignments for candidates lacking explicit bus information. + + Returns + ------- + pd.DataFrame + Combined generator dataframe with corrected bus allocations and merged existing + and potential capacities. + """ # Identify reallocations of radial wind farms using LAYER_POTENTIAL idx = ["location", "bus0", "type", "pyear", "scenario", "carrier"] generators_el = generators_e.merge( generators_l, how="outer", - on=["bus0", "type", "pyear", "scenario", "carrier"], + on=["location", "type", "pyear", "scenario", "carrier"], suffixes=("", "_l"), ) @@ -280,61 +277,72 @@ def collect_from_layer(generators_e, generators_l): "carrier.str.contains('-r') " # only radial connection "and p_nom_min != p_nom_min_l " # when EXISTING and LAYER_POTENTIAL values are inconsistent "and ~(p_nom_min.isna() and p_nom_min_l == 0)" # treat missing values and zeros as equivalent - ) + )[idx + ["p_nom_min"]] # Fix EXISTING technologies by reallocating radial to hubs - generators_e_fixed = generators_e.copy().set_index(idx) - corrections = radial_inconsistent.assign( - location=lambda x: x.bus0, carrier=lambda x: x.carrier.str.replace("-r", "-oh") - ).set_index(idx) + corrections_radial = radial_inconsistent.assign( + bus0=lambda x: x.location, carrier=lambda x: x.carrier.str.replace("-r", "-oh") + ) generators_e_fixed = ( pd.concat( [ - generators_e_fixed.drop(radial_inconsistent.set_index(idx).index), - corrections[generators_e_fixed.columns], + generators_e.set_index(idx).drop( + radial_inconsistent.set_index(idx).index + ), + corrections_radial.set_index(idx), ] ) .groupby(level=list(range(len(idx)))) .sum() + .reset_index() + .assign(carrier_mapped=lambda x: x.carrier.str.replace("h2", "dc", regex=True)) + ) + generators_l_fixed = ( + generators_l.drop(columns="p_nom_min") + .query("p_nom_max != 0") + .rename(columns={"carrier": "carrier_mapped"}) ) - # Calculate technology shares, considering PEMMDB related reallocations - tech_shares = get_shares(generators_e_fixed, values="p_nom_min") - - # Apply H2 to DC carrier mapping and get relevant shares - h2_shares = ( - tech_shares.query("carrier.str.contains('h2')") + # Combine existing capacities with potentials + # Set identical potentials for both hydrogen- and electricity-generating farms + generators = ( + generators_e_fixed.merge( + generators_l_fixed, + how="outer", + on=["location", "type", "pyear", "scenario", "carrier_mapped"], + ) .assign( - carrier_mapped=lambda x: x.carrier.str.replace( - "h2", "dc", regex=True - ).str.replace("-r", "-oh", regex=True) + p_nom_min=lambda df: df.p_nom_min.fillna(0), + carrier=lambda df: df.carrier.fillna(df.carrier_mapped), ) - .drop(columns=["tech_share", "carrier"]) - .merge(tech_shares, how="left") + .drop(columns="carrier_mapped") ) - # Apply shares to data to calculate existing capacities + # Fill missing buses generators = ( - generators_l.merge( - h2_shares, - how="outer", - left_on=["bus0", "type", "pyear", "scenario", "carrier"], - right_on=["bus0", "type", "pyear", "scenario", "carrier_mapped"], - suffixes=("_x", ""), - ) + generators.merge(nodes[["location", "HOME_NODE"]], how="left", on="location") .assign( - tech_share=lambda x: x.tech_share.fillna(1), - carrier=lambda x: x.carrier.fillna(x.carrier_x), - p_nom_min=lambda x: x.p_nom_min * x.tech_share, + bus0=lambda df: df.bus0.fillna( + df.HOME_NODE.where(df.carrier.str.contains("-r"), df.location) + ) ) - .drop(columns=["carrier_x", "tech_share", "carrier_mapped"]) + .drop(columns="HOME_NODE") ) + # Remove duplicates introduced by LAYER + generators = generators.sort_values( + by="p_nom_min", ascending=False + ).drop_duplicates(subset=idx) + return generators def load_offshore_generators( - fn: str, scenario: str, planning_horizons: list[int], countries: list[str] + fn: str, + nodes: pd.DataFrame, + scenario: str, + planning_horizons: list[int], + countries: list[str], ): """ Load offshore generators data and format data. @@ -353,6 +361,8 @@ def load_offshore_generators( ---------- fn : str Path to the Excel file containing offshore generators data. + nodes : pd.DataFrame + DataFrame containing node information. scenario : str Scenario identifier to filter the grid data. Must be one of the scenario codes: "DE" (Distributed Energy), "GA" (Global Ambition), or @@ -371,8 +381,8 @@ def load_offshore_generators( DataFrame containing the zone potentials trajectories """ column_names = { - "NODE": "location", - "OFFSHORE_NODE": "bus0", + "NODE": "bus0", + "OFFSHORE_NODE": "location", "OFFSHORE_NODE_TYPE": "type", "YEAR": "pyear", "SCENARIO": "scenario", @@ -402,16 +412,19 @@ def load_offshore_generators( } # Load data - def load_generators(sheet_name): - generators = ( - pd.read_excel( - fn, - sheet_name=sheet_name, + def load_generators(sheet_name, tech_switch=None): + generators = pd.read_excel( + fn, + sheet_name=sheet_name, + ) + if tech_switch: + generators = generators.dropna(subset=tech_switch).assign( + TECH1=lambda df: df[tech_switch] ) - .rename(columns=column_names) - .query("pyear in @planning_horizons") + generators = ( + generators.rename(columns=column_names) .replace({"scenario": scenario_dict}) - .query("scenario == @scenario") + .query("pyear in @planning_horizons and scenario == @scenario") .assign( carrier=lambda x: "offwind-" + x.carrier.str.lower().replace("_", "-", regex=True) @@ -421,7 +434,11 @@ def load_generators(sheet_name): return generators generators_e = load_generators("EXISTING") - generators_l = load_generators("LAYER_POTENTIAL") + generators_l_e = load_generators("LAYER_POTENTIAL") + generators_l_h2 = load_generators("LAYER_POTENTIAL", tech_switch="TECH2").drop( + columns="p_nom_min" + ) + generators_l = pd.concat([generators_l_e, generators_l_h2]) generators_z = load_generators("ZONE_POTENTIAL").drop( columns=["carrier", "p_nom_min"] ) @@ -431,13 +448,13 @@ def load_generators(sheet_name): ) # kEUR/MW to EUR/MW # Collect existing capacities and potentials in LAYER_POTENTIAL using H2 tech shares from EXISTING - generators = collect_from_layer(generators_e, generators_l) + generators = collect_from_layer(generators_e, generators_l, nodes) # Collect potentials trajectories in ZONE_POTENTIAL zone_trajectories = generators_z # Resolve discrepancy in DEOH002 - idx = zone_trajectories.query("bus0=='DEOH002' and pyear in [2045, 2050]").index + idx = zone_trajectories.query("location=='DEOH002' and pyear in [2045, 2050]").index zone_trajectories.loc[idx, "p_nom_max"] = ( zone_trajectories.loc[idx, "p_nom_max"] - 526 ) @@ -446,30 +463,31 @@ def load_generators(sheet_name): generators = generators.merge( generators_c, how="left", - on=["bus0", "pyear", "scenario", "type", "carrier"], + on=["bus0", "location", "pyear", "scenario", "type", "carrier"], ) - # Ensure that all cost assumptions are present - idx = generators[["capex", "opex"]].isna().any(axis=1) - if not (generators.loc[idx].p_nom_min == 0).all(): + # Validate that all required cost assumptions are defined + if generators[["capex", "opex"]].isna().any().any(): raise RuntimeError("Missing generator cost data in input dataset.") - generators = generators.dropna(subset=["capex", "opex"]) generators.loc[:, "p_nom_extendable"] = True + # Assign bus0 as location to establish home market association for radial nodes + generators.loc[:, "location"] = generators.loc[:, "bus0"] + # Rename UK in GB generators[["bus0", "location"]] = generators[["bus0", "location"]].replace( "UK", "GB", regex=True ) - zone_trajectories["bus0"] = zone_trajectories["bus0"].replace( + zone_trajectories["location"] = zone_trajectories["location"].replace( "UK", "GB", regex=True ) # Filter selected countries - generators = generators.assign(country=lambda x: x.bus0.str[:2]).query( + generators = generators.assign(country=lambda x: x.location.str[:2]).query( "country in @countries" ) zone_trajectories = zone_trajectories.assign( - country=lambda x: x.bus0.str[:2] + country=lambda x: x.location.str[:2] ).query("country in @countries") return generators, zone_trajectories @@ -510,11 +528,20 @@ def load_generators(sheet_name): generators, zone_trajectories = load_offshore_generators( snakemake.input.generators, + nodes, snakemake.params["scenario"], planning_horizons, countries, ) + # Convert country codes and retain only specified countries and offshore wind hubs + nodes[["Bus", "location", "country"]] = nodes[ + ["Bus", "location", "country"] + ].replace("UK", "GB", regex=True) + nodes = nodes.query("type != 'Radial' and country in @countries").drop( + columns="HOME_NODE" + ) + # Save data nodes.to_csv(snakemake.output.offshore_buses, index=False) grid.to_csv(snakemake.output.offshore_grid, index=False) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6db1ea5010..f24825f5ca 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3073,8 +3073,8 @@ def add_offshore_generators_tyndp( # Assign locations and index mask = offshore_generators["carrier"].str.contains("h2") - offshore_generators.loc[mask, ["bus0", "location"]] = ( - offshore_generators.loc[mask, ["bus0", "location"]] + " H2" + offshore_generators.loc[mask, "bus0"] = ( + offshore_generators.loc[mask, "bus0"] + " H2" ) offshore_generators.index = ( offshore_generators.bus0 + " " + offshore_generators.carrier @@ -3114,13 +3114,15 @@ def add_offshore_generators_tyndp( for tech_i in techs: p_max_pu.append(p_max_pu_i.rename(columns=lambda x: x + " " + tech_i)) - p_max_pu = pd.concat(p_max_pu, axis=1)[offshore_generators.index] + p_max_pu = pd.concat(p_max_pu, axis=1).reindex( + offshore_generators.index, fill_value=0 + ) # Add generators to the network n.add( "Generator", offshore_generators.index, - bus=offshore_generators.location, + bus=offshore_generators.bus0, carrier=offshore_generators.carrier, p_nom=offshore_generators.p_nom_min, p_nom_min=offshore_generators.p_nom_min, From cbba44a1ce2d449f4a28e89450a8bc69dd21b4f4 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 11:04:05 +0200 Subject: [PATCH 075/165] feat: read pecd data using location --- scripts/build_tyndp_offshore_hubs.py | 3 --- scripts/prepare_sector_network.py | 7 ++----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 8191444146..7b5195dea9 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -471,9 +471,6 @@ def load_generators(sheet_name, tech_switch=None): raise RuntimeError("Missing generator cost data in input dataset.") generators.loc[:, "p_nom_extendable"] = True - # Assign bus0 as location to establish home market association for radial nodes - generators.loc[:, "location"] = generators.loc[:, "bus0"] - # Rename UK in GB generators[["bus0", "location"]] = generators[["bus0", "location"]].replace( "UK", "GB", regex=True diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6961ccd7f3..694672c4d1 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3077,7 +3077,7 @@ def add_offshore_generators_tyndp( offshore_generators.loc[mask, "bus0"] + " H2" ) offshore_generators.index = ( - offshore_generators.bus0 + " " + offshore_generators.carrier + offshore_generators.location + " " + offshore_generators.carrier ) # Adjust capacities and costs to account for efficiency @@ -3105,7 +3105,6 @@ def add_offshore_generators_tyndp( techs = offshore_generators[ offshore_generators.carrier.str.contains(tech) ].carrier.unique() - techs = ["H2 " + tech_i if "h2" in tech_i else tech_i for tech_i in techs] with xr.open_dataset(fn) as ds: ds = ds.sel(year=pyear, bin=0, time=n.snapshots, drop=True) @@ -3114,9 +3113,7 @@ def add_offshore_generators_tyndp( for tech_i in techs: p_max_pu.append(p_max_pu_i.rename(columns=lambda x: x + " " + tech_i)) - p_max_pu = pd.concat(p_max_pu, axis=1).reindex( - offshore_generators.index, fill_value=0 - ) + p_max_pu = pd.concat(p_max_pu, axis=1).reindex(offshore_generators.index, axis=1) # Add generators to the network n.add( From 7d6eb622e2a7b819fd9acbf6fbadf7b3ad00dbff Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 12:28:14 +0200 Subject: [PATCH 076/165] refactor: use bus instead of bus0 for generators --- scripts/build_tyndp_offshore_hubs.py | 12 ++++++------ scripts/prepare_sector_network.py | 6 ++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 7b5195dea9..c86eef29f0 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -265,7 +265,7 @@ def collect_from_layer(generators_e, generators_l, nodes): and potential capacities. """ # Identify reallocations of radial wind farms using LAYER_POTENTIAL - idx = ["location", "bus0", "type", "pyear", "scenario", "carrier"] + idx = ["location", "bus", "type", "pyear", "scenario", "carrier"] generators_el = generators_e.merge( generators_l, how="outer", @@ -281,7 +281,7 @@ def collect_from_layer(generators_e, generators_l, nodes): # Fix EXISTING technologies by reallocating radial to hubs corrections_radial = radial_inconsistent.assign( - bus0=lambda x: x.location, carrier=lambda x: x.carrier.str.replace("-r", "-oh") + bus=lambda x: x.location, carrier=lambda x: x.carrier.str.replace("-r", "-oh") ) generators_e_fixed = ( pd.concat( @@ -322,7 +322,7 @@ def collect_from_layer(generators_e, generators_l, nodes): generators = ( generators.merge(nodes[["location", "HOME_NODE"]], how="left", on="location") .assign( - bus0=lambda df: df.bus0.fillna( + bus=lambda df: df.bus.fillna( df.HOME_NODE.where(df.carrier.str.contains("-r"), df.location) ) ) @@ -381,7 +381,7 @@ def load_offshore_generators( DataFrame containing the zone potentials trajectories """ column_names = { - "NODE": "bus0", + "NODE": "bus", "OFFSHORE_NODE": "location", "OFFSHORE_NODE_TYPE": "type", "YEAR": "pyear", @@ -463,7 +463,7 @@ def load_generators(sheet_name, tech_switch=None): generators = generators.merge( generators_c, how="left", - on=["bus0", "location", "pyear", "scenario", "type", "carrier"], + on=["bus", "location", "pyear", "scenario", "type", "carrier"], ) # Validate that all required cost assumptions are defined @@ -472,7 +472,7 @@ def load_generators(sheet_name, tech_switch=None): generators.loc[:, "p_nom_extendable"] = True # Rename UK in GB - generators[["bus0", "location"]] = generators[["bus0", "location"]].replace( + generators[["bus", "location"]] = generators[["bus", "location"]].replace( "UK", "GB", regex=True ) zone_trajectories["location"] = zone_trajectories["location"].replace( diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 694672c4d1..a4413217dd 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3073,9 +3073,7 @@ def add_offshore_generators_tyndp( # Assign locations and index mask = offshore_generators["carrier"].str.contains("h2") - offshore_generators.loc[mask, "bus0"] = ( - offshore_generators.loc[mask, "bus0"] + " H2" - ) + offshore_generators.loc[mask, "bus"] = offshore_generators.loc[mask, "bus"] + " H2" offshore_generators.index = ( offshore_generators.location + " " + offshore_generators.carrier ) @@ -3119,7 +3117,7 @@ def add_offshore_generators_tyndp( n.add( "Generator", offshore_generators.index, - bus=offshore_generators.bus0, + bus=offshore_generators.bus, carrier=offshore_generators.carrier, p_nom=offshore_generators.p_nom_min, p_nom_min=offshore_generators.p_nom_min, From 41f0d86ae60336ee9a6b78cfa86803dbf08a89f6 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 12:31:06 +0200 Subject: [PATCH 077/165] fix: correct typo in h2 offshore grid --- scripts/prepare_sector_network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index a4413217dd..142c0e2489 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3294,7 +3294,8 @@ def add_offshore_grid_tyndp( bus0=offshore_grid_h2.bus0, bus1=offshore_grid_h2.bus1, p_nom_extendable=offshore_grid_h2.p_nom_extendable, - p_nom=offshore_grid_h2.p_nom, + p_nom=offshore_grid_h2.p_nom_min, + p_nom_min=offshore_grid_h2.p_nom_min, p_min_pu=offshore_grid_h2.p_min_pu, p_max_pu=offshore_grid_h2.p_max_pu, capital_cost=offshore_grid_h2.capital_cost, From 3f3305d4f26dad9010d96f82619106b0a1fa2e43 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 12:46:01 +0200 Subject: [PATCH 078/165] fix: add country to rename for electrolysers --- scripts/build_tyndp_offshore_hubs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index c86eef29f0..9b91ec73a5 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -230,9 +230,9 @@ def load_offshore_electrolysers( ) # kEUR/MW to EUR/MW # rename UK in GB - electrolysers[["bus0", "bus1"]] = electrolysers[["bus0", "bus1"]].replace( - "UK", "GB", regex=True - ) + electrolysers[["bus0", "bus1", "country"]] = electrolysers[ + ["bus0", "bus1", "country"] + ].replace("UK", "GB", regex=True) # filter selected countries electrolysers = electrolysers.query("country in @countries") From af6791809924af8bd5ee689aa16c1fb954a9eac7 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 13:06:34 +0200 Subject: [PATCH 079/165] feat: improve constraint formulation to account for refactoring --- scripts/solve_network.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index babc140e10..d2d565a9bc 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1156,15 +1156,16 @@ def add_offshore_hubs_constraint( ext_i = n.generators.p_nom_extendable off_i = n.generators.index.str.contains("offwind") + gens = n.generators.assign(zone=lambda df: df.index.str.split().str[0]) - off_gens_i = n.generators.loc[(off_i) & (ext_i)].index - grouper_ext = n.generators.loc[off_gens_i].bus.map(n.buses.location) + off_gens_i = gens.loc[(off_i) & (ext_i)].index + grouper_ext = gens.loc[off_gens_i].zone idx = pd.Index(set(limit.index).intersection(grouper_ext)) eff = eff.reindex(off_gens_i, fill_value=1) lhs = (p_nom.loc[off_gens_i] / eff).groupby(grouper_ext).sum().loc[idx] - existing = n.generators.loc[(off_i) & ~(ext_i), "p_nom"] - grouper = n.generators.loc[existing.index].bus.map(n.buses.location) + existing = gens.loc[(off_i) & ~(ext_i), "p_nom"] + grouper = gens.loc[existing.index].zone existing = existing.groupby(grouper).sum().reindex(idx, fill_value=0) rhs = limit.loc[idx] - existing From 67c84ced3e8dcb69d0b06567766b8fd862baa06c Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 19 Jun 2025 13:09:37 +0200 Subject: [PATCH 080/165] bugfix: fix year dimension for 2035 an 2045 steps --- scripts/build_renewable_profiles_pecd.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 68fa9acf85..39197fb9c5 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -54,14 +54,13 @@ logger.info( f"Extract PECD capacity factor time series for year {year} for technology {technology}..." ) - + year_i = year if int(year) not in [2030, 2040, 2050]: year = np.clip(10 * (year // 10), 2030, 2050) logger.warning( "Planning horizon doesn't match available TYNDP PECD data. " f"Falling back to previous available year {year}." ) - year_i = year if year == 2050: logger.warning( "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." From cda01db1f2d3f8b662b2edc9dab638fc2a0be934 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 13:53:47 +0200 Subject: [PATCH 081/165] fix: adjust layer constraint to account for existing capacities --- scripts/solve_network.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index d2d565a9bc..fb6085b493 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1130,8 +1130,15 @@ def add_offshore_hubs_constraint( offshore_zone_trajectories_fn: str Path to the dataFrame containing the offshore zone potentials trajectories """ + ext_i = n.generators.p_nom_extendable + gens = n.generators.assign( + layer=lambda df: df.index.str.replace(r"-\d{4}$", "-2040", regex=True), + zone=lambda df: df.index.str.split().str[0], + ) + # Constraint DC / H2 expansion on the same layer - h2_gens = n.generators.loc[n.generators.carrier.str.contains("h2")] + h2_i = gens.carrier.str.contains("h2") + h2_gens = gens.loc[(h2_i) & (ext_i)] h2_gens_i = h2_gens.index dc_gens_i = h2_gens_i.str.replace("h2", "dc").str.replace(" H2", "") @@ -1141,8 +1148,13 @@ def add_offshore_hubs_constraint( eff = off_electrolysers.loc[h2_gens.bus].set_index(h2_gens_i).efficiency p_nom = n.model["Generator-p_nom"].rename({"Generator-ext": "Generator"}) + existing_l = gens.loc[(h2_i) & ~(ext_i), "p_nom"] + grouper_l = gens.loc[h2_i].layer + existing_l = existing_l.groupby(grouper_l).sum().reindex(h2_gens_i, fill_value=0) + existing_l.index = dc_gens_i + lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff - rhs = n.generators.loc[dc_gens_i].p_nom_max + rhs = gens.loc[dc_gens_i].p_nom_max - existing_l if not lhs.empty: n.model.add_constraints(lhs <= rhs, name="Generator-off_h2_dc_pot") @@ -1154,9 +1166,7 @@ def add_offshore_hubs_constraint( .p_nom_max ) - ext_i = n.generators.p_nom_extendable - off_i = n.generators.index.str.contains("offwind") - gens = n.generators.assign(zone=lambda df: df.index.str.split().str[0]) + off_i = gens.index.str.contains("offwind") off_gens_i = gens.loc[(off_i) & (ext_i)].index grouper_ext = gens.loc[off_gens_i].zone @@ -1164,10 +1174,10 @@ def add_offshore_hubs_constraint( eff = eff.reindex(off_gens_i, fill_value=1) lhs = (p_nom.loc[off_gens_i] / eff).groupby(grouper_ext).sum().loc[idx] - existing = gens.loc[(off_i) & ~(ext_i), "p_nom"] - grouper = gens.loc[existing.index].zone - existing = existing.groupby(grouper).sum().reindex(idx, fill_value=0) - rhs = limit.loc[idx] - existing + existing_z = gens.loc[(off_i) & ~(ext_i), "p_nom"] + grouper_z = gens.loc[existing_z.index].zone + existing_z = existing_z.groupby(grouper_z).sum().reindex(idx, fill_value=0) + rhs = limit.loc[idx] - existing_z n.model.add_constraints(lhs <= rhs, name="Generator-off_zone_pot") From 27a698d5e77ff1dd5bf07f5aa15224611222b1ff Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 16:09:11 +0200 Subject: [PATCH 082/165] refactor: rename variables to align with the constraints standards --- scripts/solve_network.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index fb6085b493..b4fb55faf5 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1134,7 +1134,7 @@ def add_offshore_hubs_constraint( gens = n.generators.assign( layer=lambda df: df.index.str.replace(r"-\d{4}$", "-2040", regex=True), zone=lambda df: df.index.str.split().str[0], - ) + ).rename_axis("Generator-ext") # Constraint DC / H2 expansion on the same layer h2_i = gens.carrier.str.contains("h2") @@ -1146,14 +1146,13 @@ def add_offshore_hubs_constraint( n.links.index.str.contains("Offshore Electrolysis") ].set_index("bus1") eff = off_electrolysers.loc[h2_gens.bus].set_index(h2_gens_i).efficiency - p_nom = n.model["Generator-p_nom"].rename({"Generator-ext": "Generator"}) + p_nom = n.model["Generator-p_nom"] + lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff existing_l = gens.loc[(h2_i) & ~(ext_i), "p_nom"] grouper_l = gens.loc[h2_i].layer existing_l = existing_l.groupby(grouper_l).sum().reindex(h2_gens_i, fill_value=0) existing_l.index = dc_gens_i - - lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff rhs = gens.loc[dc_gens_i].p_nom_max - existing_l if not lhs.empty: @@ -1169,7 +1168,7 @@ def add_offshore_hubs_constraint( off_i = gens.index.str.contains("offwind") off_gens_i = gens.loc[(off_i) & (ext_i)].index - grouper_ext = gens.loc[off_gens_i].zone + grouper_ext = gens.loc[off_gens_i].zone.rename("Generator-ext") idx = pd.Index(set(limit.index).intersection(grouper_ext)) eff = eff.reindex(off_gens_i, fill_value=1) lhs = (p_nom.loc[off_gens_i] / eff).groupby(grouper_ext).sum().loc[idx] From 74c8ef9e989af9570287359927003306b0d5956a Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 16:48:32 +0200 Subject: [PATCH 083/165] feat: introduce bin in index for compatibility with add_brownfield --- scripts/add_brownfield.py | 3 ++- scripts/prepare_sector_network.py | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 10cef2bbc7..aec98f8c59 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -286,7 +286,8 @@ def adjust_renewable_profiles(n, input_profiles, params, year): p_max_pu = p_max_pu.groupby(snapshotmaps).mean() # replace renewable time series - n.generators_t.p_max_pu.loc[:, p_max_pu.columns] = p_max_pu + idx = n.generators[n.generators.carrier == carrier].index + n.generators_t.p_max_pu.loc[:, p_max_pu[idx].columns] = p_max_pu[idx] def update_heat_pump_efficiency(n: pypsa.Network, n_p: pypsa.Network, year: int): diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 142c0e2489..e55a7b729b 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3075,7 +3075,7 @@ def add_offshore_generators_tyndp( mask = offshore_generators["carrier"].str.contains("h2") offshore_generators.loc[mask, "bus"] = offshore_generators.loc[mask, "bus"] + " H2" offshore_generators.index = ( - offshore_generators.location + " " + offshore_generators.carrier + offshore_generators.location + " 0 " + offshore_generators.carrier ) # Adjust capacities and costs to account for efficiency @@ -3105,11 +3105,13 @@ def add_offshore_generators_tyndp( ].carrier.unique() with xr.open_dataset(fn) as ds: - ds = ds.sel(year=pyear, bin=0, time=n.snapshots, drop=True) - p_max_pu_i = ds["profile"].to_pandas() + ds = ds.stack(bus_bin=["bus", "bin"]) + p_max_pu_i = ds["profile"].sel(year=pyear, time=n.snapshots).to_pandas() for tech_i in techs: - p_max_pu.append(p_max_pu_i.rename(columns=lambda x: x + " " + tech_i)) + p_max_pu_t = p_max_pu_i.copy() + p_max_pu_t.columns = p_max_pu_t.columns.map(flatten) + f" {tech_i}" + p_max_pu.append(p_max_pu_t) p_max_pu = pd.concat(p_max_pu, axis=1).reindex(offshore_generators.index, axis=1) From 4df0c512d6a10c416f81192c1304460287bab25a Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 17:01:44 +0200 Subject: [PATCH 084/165] feat: add colors for new carriers --- config/plotting.default.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/plotting.default.yaml b/config/plotting.default.yaml index 1dfda4a009..4e8c1e10dc 100644 --- a/config/plotting.default.yaml +++ b/config/plotting.default.yaml @@ -275,6 +275,14 @@ plotting: offwind-float: "#b5e2fa" offshore wind (Float): "#b5e2fa" offshore wind float: "#b5e2fa" + offwind-ac-fb-r: "#d0e5f7" + offwind-ac-fl-r: "#a8c5e8" + offwind-dc-fb-r: "#b8daf5" + offwind-dc-fl-r: "#85b8e6" + offwind-dc-fb-oh: "#8fbfec" + offwind-dc-fl-oh: "#5a9bd9" + offwind-h2-fb-oh: "#c47dbd" + offwind-h2-fl-oh: "#9d4d96" # water hydro: '#298c81' hydro reservoir: '#298c81' From 061fec5557fd3e7b74ffd472d04a16d37c73abe5 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 19 Jun 2025 17:04:57 +0200 Subject: [PATCH 085/165] fix: remove OH nodes from the list used for distribution --- scripts/prepare_sector_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index e55a7b729b..1e13a347f0 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1649,7 +1649,9 @@ def insert_electricity_distribution_grid( - Micro-CHP units """ - nodes = n.buses.query("carrier == 'AC' and not index.str.contains('DRES')").index + nodes = n.buses.query( + "carrier == 'AC' and not index.str.contains('DRES') and not index.str.contains('OH')" + ).index n.add( "Bus", From 22d78349b77c3bf9cb6923ebe392e8ff7ccb4853 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 20 Jun 2025 10:33:03 +0200 Subject: [PATCH 086/165] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniel Rüdt <117752024+daniel-rdt@users.noreply.github.com> --- scripts/add_brownfield.py | 5 ++--- scripts/prepare_sector_network.py | 2 ++ scripts/solve_network.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index aec98f8c59..fba5b42a1d 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -136,12 +136,11 @@ def add_brownfield( off_capacity - already_existing.reindex(index=off_capacity.index).fillna(0) ).clip(lower=0) + # this should in theory never be negative. We will still clip to account for rounding errors remaining_potential = ( off_potential - already_existing.reindex(index=off_capacity.index).fillna(0) - ).clip( - lower=0 - ) # this should anyway never be negative. We will still clip to account for rounding errors + ).clip(lower=0) c.df.loc[off_i, ["p_nom_min", "p_nom"]] = remaining_capacity c.df.loc[off_i, "p_nom_max"] = remaining_potential diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1e13a347f0..6aa3e5f05c 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3247,6 +3247,8 @@ def add_offshore_grid_tyndp( Modifies the network object in-place by adding offshore grid links. + Notes + ----- The capital costs are calculated as: (annuity_factor * capex + opex) * nyears diff --git a/scripts/solve_network.py b/scripts/solve_network.py index b4fb55faf5..4caf6a7942 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1132,7 +1132,7 @@ def add_offshore_hubs_constraint( """ ext_i = n.generators.p_nom_extendable gens = n.generators.assign( - layer=lambda df: df.index.str.replace(r"-\d{4}$", "-2040", regex=True), + layer=lambda df: df.index.str.replace(r"-\d{4}$", f"-{planning_horizons}", regex=True), zone=lambda df: df.index.str.split().str[0], ).rename_axis("Generator-ext") @@ -1222,7 +1222,7 @@ def extra_functionality( planning_horizons : str, optional The current planning horizon year or None in perfect foresight offshore_zone_trajectories_fn: str, optional - Path to the dataFrame containing the offshore zone potentials trajectories + Path to the DataFrame containing the offshore zone potentials trajectories Collects supplementary constraints which will be passed to ``pypsa.optimization.optimize``. @@ -1350,7 +1350,7 @@ def solve_network( planning_horizons : str, optional The current planning horizon year or None in perfect foresight offshore_zone_trajectories_fn : str, optional - Path to dataFrame containing the offshore zone potentials trajectories + Path to DataFrame containing the offshore zone potentials trajectories **kwargs Additional keyword arguments passed to the solver From cd94850efb40cb30a866c8cca0d37a700a24dff0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 08:33:44 +0000 Subject: [PATCH 087/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/solve_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 4caf6a7942..6fca49a856 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1132,7 +1132,9 @@ def add_offshore_hubs_constraint( """ ext_i = n.generators.p_nom_extendable gens = n.generators.assign( - layer=lambda df: df.index.str.replace(r"-\d{4}$", f"-{planning_horizons}", regex=True), + layer=lambda df: df.index.str.replace( + r"-\d{4}$", f"-{planning_horizons}", regex=True + ), zone=lambda df: df.index.str.split().str[0], ).rename_axis("Generator-ext") From da80a61e429cd012d938c2becb74943967539860 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 20 Jun 2025 10:45:47 +0200 Subject: [PATCH 088/165] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniel Rüdt <117752024+daniel-rdt@users.noreply.github.com> --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index de62c4c578..b8fb4101b8 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1077,7 +1077,7 @@ def load_cutout( def make_index(c, cname0="bus0", cname1="bus1", prefix="", connector="->", suffix=""): idx = [prefix, c[cname0], connector, c[cname1], suffix] - idx = [i for i in idx if i != ""] + idx = [i for i in idx if i] return " ".join(idx) From fe4e9ed5f08efe83fc371952a95b6f2d4d47fa4d Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 20 Jun 2025 09:37:14 +0200 Subject: [PATCH 089/165] feat: update the set of colors for the new technologies --- config/plotting.default.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/plotting.default.yaml b/config/plotting.default.yaml index 4e8c1e10dc..223a2caf43 100644 --- a/config/plotting.default.yaml +++ b/config/plotting.default.yaml @@ -269,18 +269,18 @@ plotting: offwind-ac: "#6895dd" offshore wind (AC): "#6895dd" offshore wind ac: "#6895dd" + offwind-ac-fb-r: "#6895dd" + offwind-ac-fl-r: "#6da5e8" + offwind-dc-fb-r: "#71b5ed" offwind-dc: "#74c6f2" offshore wind (DC): "#74c6f2" offshore wind dc: "#74c6f2" + offwind-dc-fb-oh: "#74c6f2" + offwind-dc-fl-r: "#94d4f6" offwind-float: "#b5e2fa" offshore wind (Float): "#b5e2fa" offshore wind float: "#b5e2fa" - offwind-ac-fb-r: "#d0e5f7" - offwind-ac-fl-r: "#a8c5e8" - offwind-dc-fb-r: "#b8daf5" - offwind-dc-fl-r: "#85b8e6" - offwind-dc-fb-oh: "#8fbfec" - offwind-dc-fl-oh: "#5a9bd9" + offwind-dc-fl-oh: "#b5e2fa" offwind-h2-fb-oh: "#c47dbd" offwind-h2-fl-oh: "#9d4d96" # water From 85f522024c9e7e2b8d6f9a7194290a8d0d29beea Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 20 Jun 2025 09:46:10 +0200 Subject: [PATCH 090/165] refactor: change config name for offshore hubs (apply suggestion) --- config/config.default.yaml | 2 +- config/config.tyndp.yaml | 2 +- config/test/config.tyndp.yaml | 2 +- doc/configtables/sector.csv | 2 +- rules/build_sector.smk | 12 ++++++------ rules/solve_myopic.smk | 4 ++-- scripts/add_brownfield.py | 8 ++++---- scripts/prepare_sector_network.py | 4 ++-- scripts/solve_network.py | 2 +- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index f1caf1e232..7ed1c00997 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -810,7 +810,7 @@ sector: methanol: 121 gas: 122 oil: 125 - offshore_hubs: false + offshore_hubs_tyndp: false # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry industry: diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 593fdf5b1e..24dd65acf6 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -139,7 +139,7 @@ sector: enable: true carriers: - H2 - offshore_hubs: true + offshore_hubs_tyndp: true costs: overwrites: diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 5e0b52b740..7b487d6607 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -158,7 +158,7 @@ sector: enable: true carriers: - H2 - offshore_hubs: true + offshore_hubs_tyndp: true costs: overwrites: diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv index 7a62724461..f564c63e1c 100644 --- a/doc/configtables/sector.csv +++ b/doc/configtables/sector.csv @@ -230,4 +230,4 @@ imports,,, -- limit_sense,--,"{==, <=, >=}",Sense of the limit -- price,,"{H2, NH3, methanol, gas, oil}", -- -- {carrier},currency/MWh,float,Price for importing renewable energy of carrier -offshore_hubs,--,"{true, false}",Add option for TYNDP offshore hubs \ No newline at end of file +offshore_hubs_tyndp,--,"{true, false}",Add option for TYNDP offshore hubs \ No newline at end of file diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 57d78fde2a..685e565157 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1362,7 +1362,7 @@ if config["sector"]["h2_topology_tyndp"]: "../scripts/build_tyndp_h2_imports.py" -if config["sector"]["offshore_hubs"]: +if config["sector"]["offshore_hubs_tyndp"]: rule build_tyndp_offshore_hubs: params: @@ -1431,7 +1431,7 @@ rule prepare_sector_network: ), load_source=config_provider("load", "source"), scaling_factor=config_provider("load", "scaling_factor"), - offshore_hubs=config_provider("sector", "offshore_hubs"), + offshore_hubs_tyndp=config_provider("sector", "offshore_hubs_tyndp"), input: unpack(input_profile_offwind), unpack(input_profile_pecd), @@ -1583,22 +1583,22 @@ rule prepare_sector_network: ), offshore_buses=lambda w: ( resources("offshore_buses.csv") - if config_provider("sector", "offshore_hubs")(w) + if config_provider("sector", "offshore_hubs_tyndp")(w) else [] ), offshore_grid=lambda w: ( resources("offshore_grid.csv") - if config_provider("sector", "offshore_hubs")(w) + if config_provider("sector", "offshore_hubs_tyndp")(w) else [] ), offshore_electrolysers=lambda w: ( resources("offshore_electrolysers.csv") - if config_provider("sector", "offshore_hubs")(w) + if config_provider("sector", "offshore_hubs_tyndp")(w) else [] ), offshore_generators=lambda w: ( resources("offshore_generators.csv") - if config_provider("sector", "offshore_hubs")(w) + if config_provider("sector", "offshore_hubs_tyndp")(w) else [] ), output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 57909fc17d..8cbdc33019 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -93,7 +93,7 @@ rule add_brownfield: dynamic_ptes_capacity=config_provider( "sector", "district_heating", "ptes", "dynamic_capacity" ), - offshore_hubs=config_provider("sector", "offshore_hubs"), + offshore_hubs_tyndp=config_provider("sector", "offshore_hubs_tyndp"), input: unpack(input_profile_tech_brownfield), unpack(input_profile_tech_brownfied_pecd), @@ -144,7 +144,7 @@ rule solve_sector_network_myopic: costs=resources("costs_{planning_horizons}.csv"), offshore_zone_trajectories=lambda w: ( resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs")(w) + if config_provider("sector", "offshore_hubs_tyndp")(w) else [] ), output: diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index fba5b42a1d..80b00caeb0 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -33,7 +33,7 @@ def add_brownfield( h2_retrofit=False, h2_retrofit_capacity_per_ch4=None, capacity_threshold=None, - offshore_hubs=False, + offshore_hubs_tyndp=False, ): """ Add brownfield capacity from previous network. @@ -52,7 +52,7 @@ def add_brownfield( Ratio of hydrogen to methane capacity for pipeline retrofitting capacity_threshold : float Threshold for removing assets with low capacity - offshore_hubs : bool + offshore_hubs_tyndp : bool Whether to enable offshore hubs """ logger.info(f"Preparing brownfield for the year {year}") @@ -114,7 +114,7 @@ def add_brownfield( n.import_series_from_dataframe(c.pnl[tattr], c.name, tattr) # adjust TYNDP offshore expansion by subtracting existing capacity from previous years from current year total capacity and potential - if offshore_hubs: + if offshore_hubs_tyndp: filter = {"Link": "Offshore", "Generator": "offwind"} for c in n.iterate_components(["Link", "Generator"]): off_fixed_i = c.df[ @@ -414,7 +414,7 @@ def update_dynamic_ptes_capacity( h2_retrofit=snakemake.params.H2_retrofit, h2_retrofit_capacity_per_ch4=snakemake.params.H2_retrofit_capacity_per_CH4, capacity_threshold=snakemake.params.threshold_capacity, - offshore_hubs=snakemake.params.offshore_hubs, + offshore_hubs_tyndp=snakemake.params.offshore_hubs_tyndp, ) disable_grid_expansion_if_limit_hit(n) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6aa3e5f05c..c54c4c70b6 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -80,7 +80,7 @@ def define_spatial(nodes, options, offshore_buses_fn=None, buses_h2_file=None): # offshore hubs - if options.get("offshore_hubs") and offshore_buses_fn: + if options.get("offshore_hubs_tyndp") and offshore_buses_fn: spatial.offshore_hubs = SimpleNamespace() offshore_buses = pd.read_csv(offshore_buses_fn, index_col=0) offshore_buses_h2 = offshore_buses.set_index(offshore_buses.index + " H2") @@ -7607,7 +7607,7 @@ def add_import_options( logger=logger, ) - if snakemake.params.offshore_hubs: + if snakemake.params.offshore_hubs_tyndp: add_offshore_hubs_tyndp( n=n, pyear=int(snakemake.wildcards.planning_horizons), diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 6fca49a856..ab58ab3c0a 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1281,7 +1281,7 @@ def extra_functionality( if config["sector"]["imports"]["enable"]: add_import_limit_constraint(n, snapshots) - if config["sector"]["offshore_hubs"]: + if config["sector"]["offshore_hubs_tyndp"]: add_offshore_hubs_constraint( n, int(planning_horizons), offshore_zone_trajectories_fn ) From 0c7f022a80db94ef80168f80fbcd044240e74fc9 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 20 Jun 2025 10:10:36 +0200 Subject: [PATCH 091/165] doc: improve release note --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index e64d132b9f..4f8130a30e 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,7 +13,7 @@ Release Notes **Changes** -* Introduce offshore wind hubs (https://github.com/open-energy-transition/open-tyndp/pull/54). +* Introduce TYNDP offshore wind hubs via `sector:offshore_hubs_tyndp` configuration (https://github.com/open-energy-transition/open-tyndp/pull/54). This feature implements an offshore grid topology with both electric and hydrogen infrastructure, offshore electrolysers, and detailed wind farm characteristics. Three wind farm types are supported: AC-radial, DC-radial and DC-hubs. Each is compatible with fixed-bottom or floating foundations. Wind farms connected to hubs can produce hydrogen directly through a dedicated P2G unit, while electricity can be supplied to the network or converted to hydrogen via offshore electrolysers connected to the hubs. The network includes existing capacities, with capacity expansion constrained by technological potential and evolving zone potential. * Add processing and preparation of TYNDP 2024 PECD renewable profiles instead of default renewable profiles using atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. From 3c764a11989a453c01ab4ad97634fae741bc807e Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 20 Jun 2025 10:43:17 +0200 Subject: [PATCH 092/165] refactor: use unpack to define offshore inputs in prepare_sector_network --- rules/build_sector.smk | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 685e565157..75a7edd36f 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1299,6 +1299,17 @@ def input_heat_source_power(w): } +def input_offshore_hubs(w): + if config_provider("sector", "offshore_hubs_tyndp")(w): + return { + "offshore_buses": resources("offshore_buses.csv"), + "offshore_grid": resources("offshore_grid.csv"), + "offshore_electrolysers": resources("offshore_electrolysers.csv"), + "offshore_generators": resources("offshore_generators.csv"), + } + return {} + + if config["sector"]["h2_topology_tyndp"]: rule build_tyndp_h2_network: @@ -1436,6 +1447,7 @@ rule prepare_sector_network: unpack(input_profile_offwind), unpack(input_profile_pecd), unpack(input_heat_source_power), + unpack(input_offshore_hubs), **rules.cluster_gas_network.output, **rules.build_gas_input_locations.output, snapshot_weightings=resources( @@ -1581,26 +1593,6 @@ rule prepare_sector_network: if config_provider("sector", "h2_topology_tyndp")(w) else [] ), - offshore_buses=lambda w: ( - resources("offshore_buses.csv") - if config_provider("sector", "offshore_hubs_tyndp")(w) - else [] - ), - offshore_grid=lambda w: ( - resources("offshore_grid.csv") - if config_provider("sector", "offshore_hubs_tyndp")(w) - else [] - ), - offshore_electrolysers=lambda w: ( - resources("offshore_electrolysers.csv") - if config_provider("sector", "offshore_hubs_tyndp")(w) - else [] - ), - offshore_generators=lambda w: ( - resources("offshore_generators.csv") - if config_provider("sector", "offshore_hubs_tyndp")(w) - else [] - ), output: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" From b5e19b2854c34ce9c925e1929209d34b0dd70070 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 20 Jun 2025 11:14:00 +0200 Subject: [PATCH 093/165] refactor: make filter more explicit to exclude offshore grid in add_brownfield --- scripts/add_brownfield.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 80b00caeb0..899b7ef613 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -59,7 +59,9 @@ def add_brownfield( # electric transmission grid set optimised capacities of previous as minimum n.lines.s_nom_min = n_p.lines.s_nom_opt - dc_i = n.links[(n.links.carrier == "DC") & (n.links.build_year < year)].index + dc_i = n.links[ + (n.links.carrier == "DC") & ~(n.links.index.str.contains("Offshore")) + ].index n.links.loc[dc_i, "p_nom_min"] = n_p.links.loc[dc_i, "p_nom_opt"] for c in n_p.iterate_components(["Link", "Generator", "Store"]): From 02ecdd907a2547617cad48cd86bede0dbaf9cdfd Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 20 Jun 2025 16:02:36 +0200 Subject: [PATCH 094/165] fix: account for the shared potential of hydrogen- and electricity-generating wind farms --- scripts/add_brownfield.py | 45 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 899b7ef613..288c83aa64 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -115,7 +115,9 @@ def add_brownfield( for tattr in n.component_attrs[c.name].index[selection]: n.import_series_from_dataframe(c.pnl[tattr], c.name, tattr) - # adjust TYNDP offshore expansion by subtracting existing capacity from previous years from current year total capacity and potential + # adjust TYNDP offshore expansion by subtracting existing capacity from previous years + # from current year total capacity and potential + # hydrogen- and electricity-generating wind farms share the same potential; values are adjusted accordingly if offshore_hubs_tyndp: filter = {"Link": "Offshore", "Generator": "offwind"} for c in n.iterate_components(["Link", "Generator"]): @@ -134,11 +136,50 @@ def add_brownfield( .groupby(level=0) .sum() ) + + # account for the shared potential of hydrogen- and electricity-generating wind farms + if c.name == "Generator": + h2_gens = already_existing.loc[ + already_existing.index.str.contains("h2") + ] + dc_gens = already_existing.loc[ + already_existing.index.str.contains("dc.*oh") + ] + + off_h2_gens = n.generators.loc[h2_gens.index] + off_dc_gens = n.generators.loc[dc_gens.index] + off_electrolysers = n.links.loc[ + n.links.index.str.contains("Offshore Electrolysis") + ].set_index("bus1") + eff_h2 = ( + off_electrolysers.loc[off_h2_gens.bus] + .set_index(h2_gens.index) + .efficiency + ) + eff_dc = ( + off_electrolysers.loc[off_dc_gens.bus + " H2"] + .set_index(dc_gens.index) + .efficiency + ) + + h2_to_dc = h2_gens.div(eff_h2).rename( + index=lambda x: x.replace("h2", "dc") + ) + dc_to_h2 = dc_gens.mul(eff_dc).rename( + index=lambda x: x.replace("dc", "h2") + ) + + already_existing = ( + pd.concat([already_existing, h2_to_dc, dc_to_h2]) + .groupby(level=0) + .sum() + ) + remaining_capacity = ( off_capacity - already_existing.reindex(index=off_capacity.index).fillna(0) ).clip(lower=0) - # this should in theory never be negative. We will still clip to account for rounding errors + # values should be non-negative; clipping applied to handle rounding errors remaining_potential = ( off_potential - already_existing.reindex(index=off_capacity.index).fillna(0) From e07e3df27cbe520756087be55954e6342abeb063 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 23 Jun 2025 13:47:53 +0200 Subject: [PATCH 095/165] fix: remove double counting of existing capacities --- scripts/solve_network.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index ab58ab3c0a..757da15ca3 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1149,13 +1149,9 @@ def add_offshore_hubs_constraint( ].set_index("bus1") eff = off_electrolysers.loc[h2_gens.bus].set_index(h2_gens_i).efficiency p_nom = n.model["Generator-p_nom"] - lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff - existing_l = gens.loc[(h2_i) & ~(ext_i), "p_nom"] - grouper_l = gens.loc[h2_i].layer - existing_l = existing_l.groupby(grouper_l).sum().reindex(h2_gens_i, fill_value=0) - existing_l.index = dc_gens_i - rhs = gens.loc[dc_gens_i].p_nom_max - existing_l + lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff + rhs = gens.loc[dc_gens_i].p_nom_max if not lhs.empty: n.model.add_constraints(lhs <= rhs, name="Generator-off_h2_dc_pot") From 95496043ca7a6f608ee67b099e01039fecbb1c5a Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 23 Jun 2025 16:09:28 +0200 Subject: [PATCH 096/165] fix: account for efficiency in existing for zonal constraint --- scripts/add_brownfield.py | 1 + scripts/solve_network.py | 25 ++++++++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 288c83aa64..9d67918138 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -151,6 +151,7 @@ def add_brownfield( off_electrolysers = n.links.loc[ n.links.index.str.contains("Offshore Electrolysis") ].set_index("bus1") + # ToDo Account for time-varying efficiencies across planning horizons eff_h2 = ( off_electrolysers.loc[off_h2_gens.bus] .set_index(h2_gens.index) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 757da15ca3..a31e796482 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1147,10 +1147,10 @@ def add_offshore_hubs_constraint( off_electrolysers = n.links.loc[ n.links.index.str.contains("Offshore Electrolysis") ].set_index("bus1") - eff = off_electrolysers.loc[h2_gens.bus].set_index(h2_gens_i).efficiency + eff_l = off_electrolysers.loc[h2_gens.bus].set_index(h2_gens_i).efficiency p_nom = n.model["Generator-p_nom"] - lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff + lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff_l rhs = gens.loc[dc_gens_i].p_nom_max if not lhs.empty: @@ -1168,12 +1168,23 @@ def add_offshore_hubs_constraint( off_gens_i = gens.loc[(off_i) & (ext_i)].index grouper_ext = gens.loc[off_gens_i].zone.rename("Generator-ext") idx = pd.Index(set(limit.index).intersection(grouper_ext)) - eff = eff.reindex(off_gens_i, fill_value=1) - lhs = (p_nom.loc[off_gens_i] / eff).groupby(grouper_ext).sum().loc[idx] - - existing_z = gens.loc[(off_i) & ~(ext_i), "p_nom"] + eff_z = eff_l.reindex(off_gens_i, fill_value=1) + lhs = (p_nom.loc[off_gens_i] / eff_z).groupby(grouper_ext).sum().loc[idx] + + # ToDo Account for time-varying efficiencies across planning horizons + existing_z = ( + gens.loc[(off_i) & ~(ext_i), "p_nom"] + .rename(lambda x: x.split("-2")[0] + f"-{planning_horizons}") + .groupby(level=0) + .sum() + ) grouper_z = gens.loc[existing_z.index].zone - existing_z = existing_z.groupby(grouper_z).sum().reindex(idx, fill_value=0) + existing_z = ( + (existing_z / eff_z.loc[existing_z.index]) + .groupby(grouper_z) + .sum() + .reindex(idx, fill_value=0) + ) rhs = limit.loc[idx] - existing_z n.model.add_constraints(lhs <= rhs, name="Generator-off_zone_pot") From 5862c1c0cdaad46a07ce09062fd5e63c220b1477 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 23 Jun 2025 16:51:16 +0200 Subject: [PATCH 097/165] feat: resolve DEOH002 capacity mismatch in existing capacities --- scripts/build_tyndp_offshore_hubs.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 9b91ec73a5..d3c45c1f8b 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -353,9 +353,9 @@ def load_offshore_generators( The `ZONE_POTENTIAL` sheet is considered as the source for achievable potentials for each node across all planning horizons. It establishes a nodal constraint on top of the theoretical potentials outlined by `LAYER_POTENTIAL`. - **Existing capacities** will be read from the `LAYER_POTENTIAL` sheet, utilizing technology shares specified in `EXISTING` for hydrogen-generating capacities. A discrepancy of 526 MW for `DEOH002` in 2045 (across all scenarios) is noted when comparing existing capacities with `ZONE_POTENTIAL`. It remains uncertain which of the two values is correct: 5828.55 MW from `LAYER_POTENTIAL` or 6354.55 MW from `ZONE_POTENTIAL`. Currently, the value of 5828.55 MW is used. + **Existing capacities** will be read from the `LAYER_POTENTIAL` sheet, utilizing technology shares specified in `EXISTING` for hydrogen-generating capacities. A discrepancy of 526 MW for `DEOH002` in 2040 (across all scenarios) is noted when comparing existing capacities with `ZONE_POTENTIAL`. It remains uncertain which of the two values is correct: 5828.55 MW from `LAYER_POTENTIAL` or 6354.55 MW from `ZONE_POTENTIAL`. Currently, the value of 5828.55 MW is used. - **Potentials** will be obtained from both the `LAYER_POTENTIAL` and the `ZONE_POTENTIAL` sheets. `LAYER_POTENTIAL` will establish a technology level constraint, while `ZONE_POTENTIAL` will restrict expansion across all technologies at each node. The same 526 MW discrepancy in `DEOH002` (across all planning horizons and scenarios) has been noted and needs to be addressed to ensure that existing capacities do not exceed their potential. Currently, the value `ZONE_POTENTIAL` value is corrected at 5828.55 MW. + **Potentials** will be obtained from both the `LAYER_POTENTIAL` and the `ZONE_POTENTIAL` sheets. `LAYER_POTENTIAL` will establish a technology level constraint, while `ZONE_POTENTIAL` will restrict expansion across all technologies at each node. The same 526 MW discrepancy for `DEOH002` in 2045 and 2050 (across all planning horizons and scenarios) has been noted and needs to be addressed to ensure that existing capacities do not exceed their potential. Currently, the value `ZONE_POTENTIAL` value is corrected at 5828.55 MW. Parameters ---------- @@ -454,9 +454,15 @@ def load_generators(sheet_name, tech_switch=None): zone_trajectories = generators_z # Resolve discrepancy in DEOH002 - idx = zone_trajectories.query("location=='DEOH002' and pyear in [2045, 2050]").index - zone_trajectories.loc[idx, "p_nom_max"] = ( - zone_trajectories.loc[idx, "p_nom_max"] - 526 + idx_l = generators.query( + "location=='DEOH002' and pyear == 2040 and carrier=='offwind-ac-fb-r'" + ).index + generators.loc[idx_l, "p_nom_min"] = generators.loc[idx_l, "p_nom_min"] - 526 + idx_z = zone_trajectories.query( + "location=='DEOH002' and pyear in [2045, 2050]" + ).index + zone_trajectories.loc[idx_z, "p_nom_max"] = ( + zone_trajectories.loc[idx_z, "p_nom_max"] - 526 ) # Collect cost assumptions From 1d09b5e3e8980fc6155b04e2842c746dd4d99116 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 23 Jun 2025 16:59:09 +0200 Subject: [PATCH 098/165] ci: revert temporary CO2 budget increase - tests now pass with original values --- config/test/config.tyndp.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 7b487d6607..4f4bce3083 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -33,7 +33,14 @@ enable: co2_budget: # TODO define carbon budget compliant with EU 2030 and 2050 targets - 2040: 0.25 # Temporary increase to allow CI to pass while load configurations are incomplete. + 2020: 0.720 # average emissions of 2019 to 2021 relative to 1990, CO2 excl LULUCF, EEA data, European Environment Agency. (2023a). Annual European Union greenhouse gas inventory 1990–2021 and inventory report 2023 - CRF Table. https://unfccc.int/documents/627830 + 2025: 0.648 # With additional measures (WAM) projection, CO2 excl LULUCF, European Environment Agency. (2023e). Member States’ greenhouse gas (GHG) emission projections 2023. https://www.eea.europa.eu/en/datahub/datahubitem-view/4b8d94a4-aed7-4e67-a54c-0623a50f48e8 + 2030: 0.450 # 55% reduction by 2030 (Ff55) + 2035: 0.250 + 2040: 0.100 # 90% by 2040 + 2045: 0.050 + 2050: 0.000 # climate-neutral by 2050 + electricity: base_network: tyndp-raw From b4b433be648fb653e918c0c46173c18aa41fb198 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 10:42:24 +0200 Subject: [PATCH 099/165] doc: refine documentation of discrepancy in DEOH002 --- scripts/build_tyndp_offshore_hubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index d3c45c1f8b..a17e45b1e7 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -353,7 +353,7 @@ def load_offshore_generators( The `ZONE_POTENTIAL` sheet is considered as the source for achievable potentials for each node across all planning horizons. It establishes a nodal constraint on top of the theoretical potentials outlined by `LAYER_POTENTIAL`. - **Existing capacities** will be read from the `LAYER_POTENTIAL` sheet, utilizing technology shares specified in `EXISTING` for hydrogen-generating capacities. A discrepancy of 526 MW for `DEOH002` in 2040 (across all scenarios) is noted when comparing existing capacities with `ZONE_POTENTIAL`. It remains uncertain which of the two values is correct: 5828.55 MW from `LAYER_POTENTIAL` or 6354.55 MW from `ZONE_POTENTIAL`. Currently, the value of 5828.55 MW is used. + **Existing capacities** will be read from the `LAYER_POTENTIAL` sheet, utilizing technology shares specified in `EXISTING` for hydrogen-generating capacities. A discrepancy of 526 MW for `DEOH002` in 2045 (across all scenarios) is noted when comparing existing capacities with `ZONE_POTENTIAL`. It remains uncertain which of the two values is correct: 5828.55 MW from `LAYER_POTENTIAL` or 6354.55 MW from `ZONE_POTENTIAL`. Currently, the value of 5828.55 MW is used. Additionally, the existing capacity of 3768.25 MW in 2040 exceeds the potential of 3242.25MW shown in the `LAYER_POTENTIAL` sheet. This value has been adjusted to match the maximum potential value across all scenarios. **Potentials** will be obtained from both the `LAYER_POTENTIAL` and the `ZONE_POTENTIAL` sheets. `LAYER_POTENTIAL` will establish a technology level constraint, while `ZONE_POTENTIAL` will restrict expansion across all technologies at each node. The same 526 MW discrepancy for `DEOH002` in 2045 and 2050 (across all planning horizons and scenarios) has been noted and needs to be addressed to ensure that existing capacities do not exceed their potential. Currently, the value `ZONE_POTENTIAL` value is corrected at 5828.55 MW. From 7ae7e5e5dcc7e747cd5b631dc0d559c725963b4a Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 10:44:58 +0200 Subject: [PATCH 100/165] doc: remove outdated documention from load_offshore_hubs --- scripts/build_tyndp_offshore_hubs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index a17e45b1e7..1aa4209f25 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -21,8 +21,6 @@ def load_offshore_hubs(fn: str, countries: list[str]): """ Load and process offshore hub coordinates from Excel file. - Offshore Hubs (OH) nodes are situated offshore, while Offshore Radial (OR) nodes are removed from the data. - Parameters ---------- fn : str @@ -347,7 +345,7 @@ def load_offshore_generators( """ Load offshore generators data and format data. - The `EXISTING` sheet is assumed to contain the collected existing capacities collected prior to any reallocations intended to align with the PEMMDB. This sheet appears to be excluded from the modelling exercise, except for hydrogen-generating capacities. + The `EXISTING` sheet is assumed to contain the existing capacities collected prior to any reallocations intended to align with the PEMMDB. This sheet appears to be excluded from the modelling exercise, except for hydrogen-generating capacities. The `LAYER_POTENTIAL` sheet is viewed as containing the reallocated existing capacities (excluding hydrogen-generating specific information) and the theoretical potentials per technology. Existing capacities are specified for both electricity- and hydrogen-generating offshore wind farms. Technology shares from `EXISTING` will be used to supplement the data. From 7b6d61976aa13332598cb3fab31530c8508801fa Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 10:56:01 +0200 Subject: [PATCH 101/165] refactor: replace generic RuntimeError with ValueError for invalid input --- scripts/build_tyndp_offshore_hubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 1aa4209f25..58e3cde0c4 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -472,7 +472,7 @@ def load_generators(sheet_name, tech_switch=None): # Validate that all required cost assumptions are defined if generators[["capex", "opex"]].isna().any().any(): - raise RuntimeError("Missing generator cost data in input dataset.") + raise ValueError("Missing generator cost data in input dataset.") generators.loc[:, "p_nom_extendable"] = True # Rename UK in GB From 35680caa93f32be14595523410a86393b7a26cf5 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 11:27:56 +0200 Subject: [PATCH 102/165] refactor: assign nyears before use --- scripts/prepare_sector_network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index c54c4c70b6..5ac6972949 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3038,6 +3038,7 @@ def add_offshore_generators_tyndp( profiles: dict[str, str], costs: pd.DataFrame, logger: logging.Logger, + nyears: float = 1, ): """ Add offshore generators to the network model. @@ -3063,6 +3064,8 @@ def add_offshore_generators_tyndp( Technology costs assumptions. logger : logging.Logger Logger for output messages. If None, no logging is performed. + nyears : float, default 1 + Number of years for which to scale the investment costs. Returns ------- @@ -3392,7 +3395,7 @@ def add_offshore_hubs_tyndp( # Add power production units add_offshore_generators_tyndp( - n, pyear, offshore_generators_fn, profiles, costs, logger + n, pyear, offshore_generators_fn, profiles, costs, logger, nyears ) # Add H2 production units From f821d606428d670443bc00ddf95b6a1a25023e01 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 11:47:01 +0200 Subject: [PATCH 103/165] fix: set offshore_zone_trajectories across all foresight scripts --- rules/solve_overnight.smk | 5 +++++ rules/solve_perfect.smk | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 5b78bc7254..e7fb103a2c 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -15,6 +15,11 @@ rule solve_sector_network: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" ), + offshore_zone_trajectories=lambda w: ( + resources("offshore_zone_trajectories.csv") + if config_provider("sector", "offshore_hubs_tyndp")(w) + else [] + ), output: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index ce9f905412..d57ba4d8b9 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -75,6 +75,11 @@ rule prepare_perfect_foresight: str(config_provider("scenario", "planning_horizons", 0)(w)) ) ), + offshore_zone_trajectories=lambda w: ( + resources("offshore_zone_trajectories.csv") + if config_provider("sector", "offshore_hubs_tyndp")(w) + else [] + ), output: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc" From b5fa72286dcff652db7e40da883e1615a6c026cb Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 12:45:39 +0200 Subject: [PATCH 104/165] doc: change position of comment in add_brownfield --- scripts/add_brownfield.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 9d67918138..a8adbb3d63 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -176,11 +176,11 @@ def add_brownfield( .sum() ) + # values should be non-negative; clipping applied to handle rounding errors remaining_capacity = ( off_capacity - already_existing.reindex(index=off_capacity.index).fillna(0) ).clip(lower=0) - # values should be non-negative; clipping applied to handle rounding errors remaining_potential = ( off_potential - already_existing.reindex(index=off_capacity.index).fillna(0) From 3441768981a5d9a1a0932521301fa1e9c5bb3f7a Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 12:47:04 +0200 Subject: [PATCH 105/165] feat: remove reference to config/config.yaml, as it is not used --- Snakefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Snakefile b/Snakefile index f21e0c5568..b691aa89e4 100644 --- a/Snakefile +++ b/Snakefile @@ -21,7 +21,6 @@ from scripts._helpers import ( configfile: "config/config.default.yaml" configfile: "config/plotting.default.yaml" configfile: "config/config.private.yaml" -configfile: "config/config.yaml" run = config["run"] From 7509ff1c928d67a60a18982141c2537d18c04fa4 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 13:56:14 +0200 Subject: [PATCH 106/165] fix: set offshore_zone_trajectories across all foresight scripts --- rules/solve_electricity.smk | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 9d08eed043..5abb65df50 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -13,6 +13,11 @@ rule solve_network: custom_extra_functionality=input_custom_extra_functionality, input: network=resources("networks/base_s_{clusters}_elec_{opts}.nc"), + offshore_zone_trajectories=lambda w: ( + resources("offshore_zone_trajectories.csv") + if config_provider("sector", "offshore_hubs_tyndp")(w) + else [] + ), output: network=RESULTS + "networks/base_s_{clusters}_elec_{opts}.nc", config=RESULTS + "configs/config.base_s_{clusters}_elec_{opts}.yaml", From a89a67fb1ac33d3751e9b19518d5e5df58a97866 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 14:28:26 +0200 Subject: [PATCH 107/165] refactor: add empty input for offshore files in prepare_sector_network --- rules/build_sector.smk | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 75a7edd36f..b73c9ae71d 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1300,14 +1300,15 @@ def input_heat_source_power(w): def input_offshore_hubs(w): + offshore_files = [ + "offshore_buses", + "offshore_grid", + "offshore_electrolysers", + "offshore_generators", + ] if config_provider("sector", "offshore_hubs_tyndp")(w): - return { - "offshore_buses": resources("offshore_buses.csv"), - "offshore_grid": resources("offshore_grid.csv"), - "offshore_electrolysers": resources("offshore_electrolysers.csv"), - "offshore_generators": resources("offshore_generators.csv"), - } - return {} + return {f: f"{f}.csv" for f in offshore_files} + return {f: [] for f in offshore_files} if config["sector"]["h2_topology_tyndp"]: From 5ccdcc7720448c78a35fa3ade09ca3d05083acb9 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 15:10:32 +0200 Subject: [PATCH 108/165] fix: set offshore_zone_trajectories across all foresight scripts --- rules/solve_perfect.smk | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index d57ba4d8b9..5b3f6a3c78 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -75,11 +75,6 @@ rule prepare_perfect_foresight: str(config_provider("scenario", "planning_horizons", 0)(w)) ) ), - offshore_zone_trajectories=lambda w: ( - resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs_tyndp")(w) - else [] - ), output: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc" @@ -112,6 +107,11 @@ rule solve_sector_network_perfect: "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc" ), costs=resources("costs_2030.csv"), + offshore_zone_trajectories=lambda w: ( + resources("offshore_zone_trajectories.csv") + if config_provider("sector", "offshore_hubs_tyndp")(w) + else [] + ), output: network=RESULTS + "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc", From 50b0bf586565d1ebd8d41b5bd5bf98be44753fb5 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 24 Jun 2025 17:23:53 +0200 Subject: [PATCH 109/165] fix: add empty input for offshore files in prepare_sector_network --- rules/build_sector.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index b73c9ae71d..a11c0885dd 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1307,7 +1307,7 @@ def input_offshore_hubs(w): "offshore_generators", ] if config_provider("sector", "offshore_hubs_tyndp")(w): - return {f: f"{f}.csv" for f in offshore_files} + return {f: resources(f"{f}.csv") for f in offshore_files} return {f: [] for f in offshore_files} From ea9034045617de2eedb9c09ef06f313015fe20e0 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 25 Jun 2025 14:52:32 +0200 Subject: [PATCH 110/165] refactor: refactor existing plotting routines to align conventions --- Snakefile | 12 ++++++ rules/postprocess.smk | 54 +++++++++++++-------------- scripts/plot_base_hydrogen_network.py | 2 +- scripts/plot_base_network.py | 4 +- 4 files changed, 42 insertions(+), 30 deletions(-) diff --git a/Snakefile b/Snakefile index b691aa89e4..e53cec6054 100644 --- a/Snakefile +++ b/Snakefile @@ -79,6 +79,7 @@ if config["foresight"] == "perfect": rule all: input: expand(RESULTS + "graphs/costs.svg", run=config["run"]["name"]), + expand(resources("maps/power-network.pdf")), expand( resources("maps/power-network-s-{clusters}.pdf"), run=config["run"]["name"], @@ -90,6 +91,17 @@ rule all: run=config["run"]["name"], **config["scenario"], ), + lambda w: expand( + ( + resources( + "maps/base_h2_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" + ) + if config_provider("sector", "H2_network")(w) + else [] + ), + run=config["run"]["name"], + **config["scenario"], + ), lambda w: expand( ( RESULTS diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 3d0680623f..e79c4229f4 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -68,6 +68,33 @@ if config["foresight"] != "perfect": script: "../scripts/plot_power_network.py" + rule plot_base_hydrogen_network: + params: + plotting=config_provider("plotting"), + input: + network=resources( + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" + ), + regions_onshore=resources("regions_onshore.geojson"), + output: + map=resources( + "maps/base_h2_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" + ), + threads: 1 + resources: + mem_mb=4000, + benchmark: + benchmarks( + "plot_base_hydrogen_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + ) + log: + RESULTS + + "logs/plot_base_hydrogen_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + conda: + "../envs/environment.yaml" + script: + "../scripts/plot_base_hydrogen_network.py" + rule plot_hydrogen_network: params: plotting=config_provider("plotting"), @@ -100,33 +127,6 @@ if config["foresight"] != "perfect": script: "../scripts/plot_hydrogen_network.py" - rule plot_base_hydrogen_network: - params: - plotting=config_provider("plotting"), - input: - network=resources( - "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" - ), - regions_onshore=resources("regions_onshore.geojson"), - output: - map=resources( - "maps/base_h2_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" - ), - threads: 1 - resources: - mem_mb=4000, - benchmark: - benchmarks( - "plot_base_hydrogen_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}" - ) - log: - RESULTS - + "logs/plot_base_hydrogen_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", - conda: - "../envs/environment.yaml" - script: - "../scripts/plot_base_hydrogen_network.py" - rule plot_gas_network: params: plotting=config_provider("plotting"), diff --git a/scripts/plot_base_hydrogen_network.py b/scripts/plot_base_hydrogen_network.py index 6e8184ba07..0b058ea819 100644 --- a/scripts/plot_base_hydrogen_network.py +++ b/scripts/plot_base_hydrogen_network.py @@ -228,7 +228,7 @@ def plot_h2_map_base( snakemake = mock_snakemake( "plot_base_hydrogen_network", opts="", - clusters="100", + clusters="all", sector_opts="", planning_horizons=2030, ) diff --git a/scripts/plot_base_network.py b/scripts/plot_base_network.py index 5cb1cde5f0..02720f62e8 100644 --- a/scripts/plot_base_network.py +++ b/scripts/plot_base_network.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: MIT """ -Plot base network transmission network. +Plot base transmission network. """ import geopandas as gpd @@ -16,7 +16,7 @@ if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("plot_base_network", run="tyndp-raw") + snakemake = mock_snakemake("plot_base_network") set_scenario_config(snakemake) n = pypsa.Network(snakemake.input.network) From 07771f4c1c2e395eaea16f3a3008cb72d924c2a5 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 25 Jun 2025 17:00:02 +0200 Subject: [PATCH 111/165] feat: introduce offshore plotting routines --- Snakefile | 21 +++ rules/postprocess.smk | 45 ++++++ scripts/plot_base_offshore_network.py | 217 ++++++++++++++++++++++++++ 3 files changed, 283 insertions(+) create mode 100644 scripts/plot_base_offshore_network.py diff --git a/Snakefile b/Snakefile index e53cec6054..71253c1699 100644 --- a/Snakefile +++ b/Snakefile @@ -122,6 +122,27 @@ rule all: run=config["run"]["name"], **config["scenario"], ), + lambda w: expand( + ( + resources( + "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" + ) + if config_provider("sector", "offshore_hubs_tyndp")(w) + else [] + ), + run=config["run"]["name"], + **config["scenario"], + ), + lambda w: expand( + ( + RESULTS + + "maps/base_s_{clusters}_{opts}_{sector_opts}-offshore_network_{planning_horizons}.pdf" + if config_provider("sector", "offshore_hubs_tyndp")(w) + else [] + ), + run=config["run"]["name"], + **config["scenario"], + ), lambda w: expand( ( RESULTS + "csvs/cumulative_costs.csv" diff --git a/rules/postprocess.smk b/rules/postprocess.smk index e79c4229f4..fb9b7305ef 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -127,6 +127,51 @@ if config["foresight"] != "perfect": script: "../scripts/plot_hydrogen_network.py" + rule plot_base_offshore_network: + params: + plotting=config_provider("plotting"), + expanded=False, + input: + network=resources( + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" + ), + regions_offshore=resources("regions_offshore.geojson"), + output: + map=resources( + "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" + ), + threads: 1 + resources: + mem_mb=4000, + benchmark: + benchmarks( + "plot_base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + ) + log: + RESULTS + + "logs/plot_base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + conda: + "../envs/environment.yaml" + script: + "../scripts/plot_base_offshore_network.py" + + use rule plot_base_offshore_network as plot_offshore_network with: + params: + expanded=True, + input: + network=RESULTS + + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", + output: + map=RESULTS + + "maps/base_s_{clusters}_{opts}_{sector_opts}-offshore_network_{planning_horizons}.pdf", + benchmark: + benchmarks( + "plot_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + ) + log: + RESULTS + + "logs/plot_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + rule plot_gas_network: params: plotting=config_provider("plotting"), diff --git a/scripts/plot_base_offshore_network.py b/scripts/plot_base_offshore_network.py new file mode 100644 index 0000000000..b123e9cc43 --- /dev/null +++ b/scripts/plot_base_offshore_network.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: MIT +""" +Plot offshore base transmission network. +""" + +import logging + +import geopandas as gpd +import matplotlib.pyplot as plt +import pypsa +from _helpers import configure_logging, set_scenario_config +from plot_power_network import load_projection +from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches + +plt.style.use(["ggplot"]) + + +logger = logging.getLogger(__name__) + + +def plot_offshore_map_base(network, map_opts, map_fn, expanded=False): + """ + Plots the base offshore network hydrogen and electricity capacities and offshore-hubs buses. + If expanded is enabled, the optimal capacities are plotted instead. + + Parameters + ---------- + network : pypsa.Network + PyPSA network for plotting the offshore grid. Can be either presolving or post solving. + map_opts : dict + Map options for plotting. + map_fn : str + Path to save the final map plot to. + expanded : bool, optional + Whether to plot expanded capacities. Defaults to plotting only base network (p_nom). + + Returns + ------- + None + Saves the map plot as figure. + """ + n = network.copy() + + linewidth_factor = 4e3 + + n.links.drop( + n.links.index[ + ~( + n.links.index.str.contains("Offshore H2 pipeline") + | n.links.index.str.contains("Offshore DC") + ) + ], + inplace=True, + ) + + p_nom = "p_nom_opt" if expanded else "p_nom" + # transmission capacities + links_dc = n.links[n.links.index.str.contains("Offshore DC")][p_nom] + links_h2 = n.links[n.links.index.str.contains("Offshore H2 pipeline")][p_nom] + + # set link widths + link_widths_dc = links_dc / linewidth_factor + link_widths_h2 = links_h2 / linewidth_factor + if link_widths_h2.notnull().empty and link_widths_dc.notnull().empty: + logger.info("No offshore base capacities to plot.") + return + link_widths_h2 = link_widths_h2.reindex(n.links.index).fillna(0.0) + link_widths_dc = link_widths_dc.reindex(n.links.index).fillna(0.0) + + # keep relevant buses + n.buses.drop( + n.buses.index[ + (~n.buses.carrier.isin(["AC", "DC", "H2"])) + | (n.buses.index.str.contains("Z1|Z2")) + ], + inplace=True, + ) + n_oh = n.copy() + n_oh.buses.drop( + n_oh.buses.index[~n_oh.buses.index.str.contains("OH")], inplace=True + ) + + # plot transmission network + logger.info("Plotting offshore transmission network.") + proj = load_projection(dict(name="EqualEarth")) + fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": proj}) + color_h2 = "#f081dc" + color_dc = "darkseagreen" + color_oh_nodes = "#ff29d9" + color_hm_nodes = "darkgray" + + n.plot( + geomap=True, + bus_sizes=0.05, + bus_colors=color_hm_nodes, + link_colors=color_h2, + link_widths=link_widths_h2, + branch_components=["Link"], + ax=ax, + **map_opts, + ) + + n_oh.plot( + geomap=True, + bus_sizes=0.05, + bus_colors=color_oh_nodes, + branch_components=[], + ax=ax, + **map_opts, + ) + + n.plot( + geomap=True, + bus_sizes=0, + link_colors=color_dc, + link_widths=link_widths_dc, + branch_components=["Link"], + ax=ax, + **map_opts, + ) + + sizes = [30, 10] + labels = [f"{s} GW" for s in sizes] + scale = 1e3 / 4e3 + sizes = [s * scale for s in sizes] + + legend_kw = dict( + loc="upper left", + bbox_to_anchor=(0.32, 1.13), + frameon=False, + ncol=1, + labelspacing=0.8, + handletextpad=1, + ) + + add_legend_lines( + ax, + sizes, + labels, + patch_kw=dict(color="lightgrey"), + legend_kw=legend_kw, + ) + + legend_kw = dict( + loc="upper left", + bbox_to_anchor=(0.55, 1.13), + labelspacing=0.8, + handletextpad=0, + frameon=False, + ) + + add_legend_circles( + ax, + sizes=[0.1], + labels=["Home market"], + srid=n.srid, + patch_kw=dict(facecolor=color_hm_nodes), + legend_kw=legend_kw, + ) + + legend_kw["bbox_to_anchor"] = (0.55, 1.08) + + add_legend_circles( + ax, + sizes=[0.1], + labels=["Offshore hubs"], + srid=n.srid, + patch_kw=dict(facecolor=color_oh_nodes), + legend_kw=legend_kw, + ) + + colors = [color_dc, color_h2] + labels = ["DC link", "H2 pipeline"] + + legend_kw = dict( + loc="upper left", + bbox_to_anchor=(0, 1.13), + ncol=1, + frameon=False, + ) + + add_legend_patches(ax, colors, labels, legend_kw=legend_kw) + + ax.set_facecolor("white") + + plt.savefig(map_fn, bbox_inches="tight") + plt.close() + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake( + "plot_base_offshore_network", + opts="", + clusters="all", + sector_opts="", + planning_horizons=2050, + ) + configure_logging(snakemake) + set_scenario_config(snakemake) + + n = pypsa.Network(snakemake.input.network) + + map_opts = snakemake.params.plotting["map"] + + if map_opts["boundaries"] is None: + regions = gpd.read_file(snakemake.input.regions_onshore).set_index("name") + map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] + + proj = load_projection(snakemake.params.plotting) + map_fn = snakemake.output.map + + plot_offshore_map_base(n, map_opts, map_fn, expanded=snakemake.params.expanded) From f97d360aac936e45c0de5395141d7cf41b6e2929 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 25 Jun 2025 17:05:14 +0200 Subject: [PATCH 112/165] refactor: match new naming convention for plots --- Snakefile | 4 ++-- rules/postprocess.smk | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Snakefile b/Snakefile index 71253c1699..7b27591c9d 100644 --- a/Snakefile +++ b/Snakefile @@ -105,7 +105,7 @@ rule all: lambda w: expand( ( RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf" + + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-h2_network.pdf" if config_provider("sector", "H2_network")(w) else [] ), @@ -136,7 +136,7 @@ rule all: lambda w: expand( ( RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}-offshore_network_{planning_horizons}.pdf" + + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network.pdf" if config_provider("sector", "offshore_hubs_tyndp")(w) else [] ), diff --git a/rules/postprocess.smk b/rules/postprocess.smk index fb9b7305ef..9cd62cbad8 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -110,7 +110,7 @@ if config["foresight"] != "perfect": ), output: map=RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf", + + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-h2_network.pdf", threads: 2 resources: mem_mb=10000, @@ -163,7 +163,7 @@ if config["foresight"] != "perfect": + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", output: map=RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}-offshore_network_{planning_horizons}.pdf", + + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network.pdf", benchmark: benchmarks( "plot_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}" From ca4eb66ef85d9af659318944ff43665420d38c87 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 25 Jun 2025 17:09:58 +0200 Subject: [PATCH 113/165] refactor: remove the default reference to base network for offshore plotting routine --- rules/postprocess.smk | 2 +- ..._offshore_network.py => plot_offshore_network.py} | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) rename scripts/{plot_base_offshore_network.py => plot_offshore_network.py} (93%) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 9cd62cbad8..d9f6b872cc 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -153,7 +153,7 @@ if config["foresight"] != "perfect": conda: "../envs/environment.yaml" script: - "../scripts/plot_base_offshore_network.py" + "../scripts/plot_offshore_network.py" use rule plot_base_offshore_network as plot_offshore_network with: params: diff --git a/scripts/plot_base_offshore_network.py b/scripts/plot_offshore_network.py similarity index 93% rename from scripts/plot_base_offshore_network.py rename to scripts/plot_offshore_network.py index b123e9cc43..92ce8f0b44 100644 --- a/scripts/plot_base_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: MIT """ -Plot offshore base transmission network. +Plot offshore transmission network. """ import logging @@ -20,9 +20,9 @@ logger = logging.getLogger(__name__) -def plot_offshore_map_base(network, map_opts, map_fn, expanded=False): +def plot_offshore_map(network, map_opts, map_fn, expanded=False): """ - Plots the base offshore network hydrogen and electricity capacities and offshore-hubs buses. + Plots the offshore network hydrogen and electricity capacities and offshore-hubs buses. If expanded is enabled, the optimal capacities are plotted instead. Parameters @@ -64,7 +64,7 @@ def plot_offshore_map_base(network, map_opts, map_fn, expanded=False): link_widths_dc = links_dc / linewidth_factor link_widths_h2 = links_h2 / linewidth_factor if link_widths_h2.notnull().empty and link_widths_dc.notnull().empty: - logger.info("No offshore base capacities to plot.") + logger.info("No offshore capacities to plot.") return link_widths_h2 = link_widths_h2.reindex(n.links.index).fillna(0.0) link_widths_dc = link_widths_dc.reindex(n.links.index).fillna(0.0) @@ -194,7 +194,7 @@ def plot_offshore_map_base(network, map_opts, map_fn, expanded=False): from _helpers import mock_snakemake snakemake = mock_snakemake( - "plot_base_offshore_network", + "plot_offshore_network", opts="", clusters="all", sector_opts="", @@ -214,4 +214,4 @@ def plot_offshore_map_base(network, map_opts, map_fn, expanded=False): proj = load_projection(snakemake.params.plotting) map_fn = snakemake.output.map - plot_offshore_map_base(n, map_opts, map_fn, expanded=snakemake.params.expanded) + plot_offshore_map(n, map_opts, map_fn, expanded=snakemake.params.expanded) From 71d758aa3ea099c815990385cdaf20bf32552719 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 25 Jun 2025 17:56:57 +0200 Subject: [PATCH 114/165] refactor: split offshore plotting routine using a carrier wildcard --- Snakefile | 6 ++- config/plotting.default.yaml | 4 ++ rules/postprocess.smk | 12 +++--- scripts/plot_base_hydrogen_network.py | 2 +- scripts/plot_offshore_network.py | 61 ++++++++++++--------------- 5 files changed, 42 insertions(+), 43 deletions(-) diff --git a/Snakefile b/Snakefile index 7b27591c9d..656f92d0cd 100644 --- a/Snakefile +++ b/Snakefile @@ -125,23 +125,25 @@ rule all: lambda w: expand( ( resources( - "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" + "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.pdf" ) if config_provider("sector", "offshore_hubs_tyndp")(w) else [] ), run=config["run"]["name"], **config["scenario"], + carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), ), lambda w: expand( ( RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network.pdf" + + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network_{carrier}.pdf" if config_provider("sector", "offshore_hubs_tyndp")(w) else [] ), run=config["run"]["name"], **config["scenario"], + carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), ), lambda w: expand( ( diff --git a/config/plotting.default.yaml b/config/plotting.default.yaml index 223a2caf43..f0b64a8844 100644 --- a/config/plotting.default.yaml +++ b/config/plotting.default.yaml @@ -241,6 +241,10 @@ plotting: - 200 - 100 branch_sizes: + offshore_maps: + bus_carriers: + - DC + - H2 nice_names: OCGT: "Open-Cycle Gas" diff --git a/rules/postprocess.smk b/rules/postprocess.smk index d9f6b872cc..30e7bfc202 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -138,18 +138,18 @@ if config["foresight"] != "perfect": regions_offshore=resources("regions_offshore.geojson"), output: map=resources( - "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" + "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.pdf" ), threads: 1 resources: mem_mb=4000, benchmark: benchmarks( - "plot_base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + "plot_base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}" ) log: RESULTS - + "logs/plot_base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + + "logs/plot_base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.log", conda: "../envs/environment.yaml" script: @@ -163,14 +163,14 @@ if config["foresight"] != "perfect": + "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc", output: map=RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network.pdf", + + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network_{carrier}.pdf", benchmark: benchmarks( - "plot_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}" + "plot_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}" ) log: RESULTS - + "logs/plot_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.log", + + "logs/plot_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.log", rule plot_gas_network: params: diff --git a/scripts/plot_base_hydrogen_network.py b/scripts/plot_base_hydrogen_network.py index 0b058ea819..8e3bc2248b 100644 --- a/scripts/plot_base_hydrogen_network.py +++ b/scripts/plot_base_hydrogen_network.py @@ -164,7 +164,7 @@ def plot_h2_map_base( sizes = [30, 10] labels = [f"{s} GW" for s in sizes] - scale = 1e3 / 4e3 + scale = 1e3 / linewidth_factor sizes = [s * scale for s in sizes] legend_kw = dict( diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index 92ce8f0b44..bbd00b3ede 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) -def plot_offshore_map(network, map_opts, map_fn, expanded=False): +def plot_offshore_map(network, map_opts, map_fn, carrier="DC", expanded=False): """ Plots the offshore network hydrogen and electricity capacities and offshore-hubs buses. If expanded is enabled, the optimal capacities are plotted instead. @@ -33,6 +33,8 @@ def plot_offshore_map(network, map_opts, map_fn, expanded=False): Map options for plotting. map_fn : str Path to save the final map plot to. + carrier : str, optional + Carrier to plot expanded : bool, optional Whether to plot expanded capacities. Defaults to plotting only base network (p_nom). @@ -43,36 +45,30 @@ def plot_offshore_map(network, map_opts, map_fn, expanded=False): """ n = network.copy() - linewidth_factor = 4e3 + linewidth_factor = 1e4 + + mask = {"DC": "Offshore DC", "H2": "Offshore H2 pipeline"} n.links.drop( - n.links.index[ - ~( - n.links.index.str.contains("Offshore H2 pipeline") - | n.links.index.str.contains("Offshore DC") - ) - ], + n.links.index[~(n.links.index.str.contains(mask[carrier]))], inplace=True, ) p_nom = "p_nom_opt" if expanded else "p_nom" # transmission capacities - links_dc = n.links[n.links.index.str.contains("Offshore DC")][p_nom] - links_h2 = n.links[n.links.index.str.contains("Offshore H2 pipeline")][p_nom] + links = n.links[n.links.index.str.contains(mask[carrier])][p_nom] # set link widths - link_widths_dc = links_dc / linewidth_factor - link_widths_h2 = links_h2 / linewidth_factor - if link_widths_h2.notnull().empty and link_widths_dc.notnull().empty: - logger.info("No offshore capacities to plot.") + link_widths = links / linewidth_factor + if link_widths.notnull().empty: + logger.info(f"No offshore capacities for {carrier}, skipping plot.") return - link_widths_h2 = link_widths_h2.reindex(n.links.index).fillna(0.0) - link_widths_dc = link_widths_dc.reindex(n.links.index).fillna(0.0) + link_widths = link_widths.reindex(n.links.index).fillna(0.0) # keep relevant buses n.buses.drop( n.buses.index[ - (~n.buses.carrier.isin(["AC", "DC", "H2"])) + (~n.buses.carrier.isin(["AC"] + [carrier])) | (n.buses.index.str.contains("Z1|Z2")) ], inplace=True, @@ -88,6 +84,7 @@ def plot_offshore_map(network, map_opts, map_fn, expanded=False): fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": proj}) color_h2 = "#f081dc" color_dc = "darkseagreen" + color = color_dc if carrier == "DC" else color_h2 color_oh_nodes = "#ff29d9" color_hm_nodes = "darkgray" @@ -95,8 +92,8 @@ def plot_offshore_map(network, map_opts, map_fn, expanded=False): geomap=True, bus_sizes=0.05, bus_colors=color_hm_nodes, - link_colors=color_h2, - link_widths=link_widths_h2, + link_colors=color, + link_widths=link_widths, branch_components=["Link"], ax=ax, **map_opts, @@ -111,19 +108,9 @@ def plot_offshore_map(network, map_opts, map_fn, expanded=False): **map_opts, ) - n.plot( - geomap=True, - bus_sizes=0, - link_colors=color_dc, - link_widths=link_widths_dc, - branch_components=["Link"], - ax=ax, - **map_opts, - ) - sizes = [30, 10] labels = [f"{s} GW" for s in sizes] - scale = 1e3 / 4e3 + scale = 1e3 / linewidth_factor sizes = [s * scale for s in sizes] legend_kw = dict( @@ -171,8 +158,7 @@ def plot_offshore_map(network, map_opts, map_fn, expanded=False): legend_kw=legend_kw, ) - colors = [color_dc, color_h2] - labels = ["DC link", "H2 pipeline"] + label = "DC link" if carrier == "DC" else "H2 pipeline" legend_kw = dict( loc="upper left", @@ -181,7 +167,7 @@ def plot_offshore_map(network, map_opts, map_fn, expanded=False): frameon=False, ) - add_legend_patches(ax, colors, labels, legend_kw=legend_kw) + add_legend_patches(ax, color, label, legend_kw=legend_kw) ax.set_facecolor("white") @@ -199,6 +185,7 @@ def plot_offshore_map(network, map_opts, map_fn, expanded=False): clusters="all", sector_opts="", planning_horizons=2050, + carrier="DC", ) configure_logging(snakemake) set_scenario_config(snakemake) @@ -214,4 +201,10 @@ def plot_offshore_map(network, map_opts, map_fn, expanded=False): proj = load_projection(snakemake.params.plotting) map_fn = snakemake.output.map - plot_offshore_map(n, map_opts, map_fn, expanded=snakemake.params.expanded) + plot_offshore_map( + n, + map_opts, + map_fn, + carrier=snakemake.wildcards.carrier, + expanded=snakemake.params.expanded, + ) From 2eca4602caadc60a24ae23ed3f4f0fcd13b30384 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 11:37:19 +0200 Subject: [PATCH 115/165] fix: attach h2 piplines to the correct buses --- scripts/plot_offshore_network.py | 6 ++++-- scripts/prepare_sector_network.py | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index bbd00b3ede..884727940d 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -66,10 +66,12 @@ def plot_offshore_map(network, map_opts, map_fn, carrier="DC", expanded=False): link_widths = link_widths.reindex(n.links.index).fillna(0.0) # keep relevant buses + bus_carriers = [carrier] + (["AC"] if carrier == "DC" else []) n.buses.drop( n.buses.index[ - (~n.buses.carrier.isin(["AC"] + [carrier])) - | (n.buses.index.str.contains("Z1|Z2")) + (~n.buses.carrier.isin(bus_carriers)) + | (n.buses.index.str.contains("Z1")) + | (n.buses.index.str.contains("DRES")) ], inplace=True, ) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 5ac6972949..b7a711c0e7 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3288,6 +3288,14 @@ def add_offshore_grid_tyndp( # Add H2 pipeline connections offshore_grid_h2 = offshore_grid.query("carrier=='H2'").copy() + offshore_grid_h2 = offshore_grid_h2.assign( + bus0=lambda df: np.where( + df.bus0.str.contains("OH"), df.bus0 + " H2", df.bus0.str[:2] + " H2 Z2" + ), + bus1=lambda df: np.where( + df.bus1.str.contains("OH"), df.bus1 + " H2", df.bus1.str[:2] + " H2 Z2" + ), + ) offshore_grid_h2.index = offshore_grid_h2.apply( make_index, axis=1, prefix="Offshore H2 pipeline" ) From 394f4e4fe044950152a6c280f8f03e8aa8609283 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 12:05:09 +0200 Subject: [PATCH 116/165] fix: remove previous build year from efficiency calculations --- scripts/add_brownfield.py | 3 ++- scripts/solve_network.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index a8adbb3d63..fe54009780 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -149,7 +149,8 @@ def add_brownfield( off_h2_gens = n.generators.loc[h2_gens.index] off_dc_gens = n.generators.loc[dc_gens.index] off_electrolysers = n.links.loc[ - n.links.index.str.contains("Offshore Electrolysis") + (n.links.index.str.contains("Offshore Electrolysis")) + & (n.links.build_year == year) ].set_index("bus1") # ToDo Account for time-varying efficiencies across planning horizons eff_h2 = ( diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a31e796482..f755b284a9 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1145,7 +1145,8 @@ def add_offshore_hubs_constraint( dc_gens_i = h2_gens_i.str.replace("h2", "dc").str.replace(" H2", "") off_electrolysers = n.links.loc[ - n.links.index.str.contains("Offshore Electrolysis") + (n.links.index.str.contains("Offshore Electrolysis")) + & (n.links.build_year == planning_horizons) ].set_index("bus1") eff_l = off_electrolysers.loc[h2_gens.bus].set_index(h2_gens_i).efficiency p_nom = n.model["Generator-p_nom"] From 0f174c9a143cce479a062f224e954f8d74cc1d2c Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 14:50:44 +0200 Subject: [PATCH 117/165] fix: group links in plot_offshore_network --- scripts/plot_offshore_network.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index 884727940d..a488a3e61d 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -6,6 +6,7 @@ """ import logging +import re import geopandas as gpd import matplotlib.pyplot as plt @@ -20,7 +21,9 @@ logger = logging.getLogger(__name__) -def plot_offshore_map(network, map_opts, map_fn, carrier="DC", expanded=False): +def plot_offshore_map( + network, map_opts, map_fn, planning_horizons, carrier="DC", expanded=False +): """ Plots the offshore network hydrogen and electricity capacities and offshore-hubs buses. If expanded is enabled, the optimal capacities are plotted instead. @@ -33,6 +36,8 @@ def plot_offshore_map(network, map_opts, map_fn, carrier="DC", expanded=False): Map options for plotting. map_fn : str Path to save the final map plot to. + planning_horizons : int + The planning horizon year carrier : str, optional Carrier to plot expanded : bool, optional @@ -56,8 +61,12 @@ def plot_offshore_map(network, map_opts, map_fn, carrier="DC", expanded=False): p_nom = "p_nom_opt" if expanded else "p_nom" # transmission capacities - links = n.links[n.links.index.str.contains(mask[carrier])][p_nom] - + links = ( + n.links[n.links.index.str.contains(mask[carrier])][p_nom] + .rename(index=lambda x: re.sub(r"-\d{4}$", f"-{planning_horizons}", x)) + .groupby(level=0) + .sum() + ) # set link widths link_widths = links / linewidth_factor if link_widths.notnull().empty: @@ -207,6 +216,7 @@ def plot_offshore_map(network, map_opts, map_fn, carrier="DC", expanded=False): n, map_opts, map_fn, + snakemake.wildcards.planning_horizons, carrier=snakemake.wildcards.carrier, expanded=snakemake.params.expanded, ) From 39151ab06dc19e5d4b1aa35e9c6eae39688aa52a Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 15:40:38 +0200 Subject: [PATCH 118/165] feat: add a offshore transmission capacity limits --- Snakefile | 4 ++-- config/config.default.yaml | 6 +++++- config/config.tyndp.yaml | 3 ++- config/test/config.tyndp.yaml | 3 ++- doc/configtables/sector.csv | 5 ++++- rules/build_sector.smk | 7 ++++--- rules/solve_electricity.smk | 2 +- rules/solve_myopic.smk | 4 ++-- rules/solve_overnight.smk | 2 +- rules/solve_perfect.smk | 2 +- scripts/build_tyndp_offshore_hubs.py | 10 ++++++++++ scripts/prepare_sector_network.py | 2 ++ scripts/solve_network.py | 2 +- 13 files changed, 37 insertions(+), 15 deletions(-) diff --git a/Snakefile b/Snakefile index 656f92d0cd..3bdbc8eebc 100644 --- a/Snakefile +++ b/Snakefile @@ -127,7 +127,7 @@ rule all: resources( "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.pdf" ) - if config_provider("sector", "offshore_hubs_tyndp")(w) + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) else [] ), run=config["run"]["name"], @@ -138,7 +138,7 @@ rule all: ( RESULTS + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network_{carrier}.pdf" - if config_provider("sector", "offshore_hubs_tyndp")(w) + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) else [] ), run=config["run"]["name"], diff --git a/config/config.default.yaml b/config/config.default.yaml index e9f53cb9fe..fba8c781a7 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -810,7 +810,11 @@ sector: methanol: 121 gas: 122 oil: 125 - offshore_hubs_tyndp: false + offshore_hubs_tyndp: + enable: false + max_capacity: + DC: 10 + H2: 30 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry industry: diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 24dd65acf6..03d4760eb7 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -139,7 +139,8 @@ sector: enable: true carriers: - H2 - offshore_hubs_tyndp: true + offshore_hubs_tyndp: + enable: true costs: overwrites: diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 4f4bce3083..22a71857e5 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -165,7 +165,8 @@ sector: enable: true carriers: - H2 - offshore_hubs_tyndp: true + offshore_hubs_tyndp: + enable: true costs: overwrites: diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv index f564c63e1c..3057b1183f 100644 --- a/doc/configtables/sector.csv +++ b/doc/configtables/sector.csv @@ -230,4 +230,7 @@ imports,,, -- limit_sense,--,"{==, <=, >=}",Sense of the limit -- price,,"{H2, NH3, methanol, gas, oil}", -- -- {carrier},currency/MWh,float,Price for importing renewable energy of carrier -offshore_hubs_tyndp,--,"{true, false}",Add option for TYNDP offshore hubs \ No newline at end of file +offshore_hubs_tyndp,--,"{true, false}",Add options for TYNDP offshore hubs +-- enable,--,"{true, false}",Add option to include TYNDP offshore hubs +-- max_capacity,--,"{true, false}",Maximum transmission capacities between two offshore hubs +-- -- {carrier},GW,float,Maximum transmission capacity between two offshore hubs of a carrier \ No newline at end of file diff --git a/rules/build_sector.smk b/rules/build_sector.smk index a11c0885dd..6acfd6ce13 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1306,7 +1306,7 @@ def input_offshore_hubs(w): "offshore_electrolysers", "offshore_generators", ] - if config_provider("sector", "offshore_hubs_tyndp")(w): + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w): return {f: resources(f"{f}.csv") for f in offshore_files} return {f: [] for f in offshore_files} @@ -1374,13 +1374,14 @@ if config["sector"]["h2_topology_tyndp"]: "../scripts/build_tyndp_h2_imports.py" -if config["sector"]["offshore_hubs_tyndp"]: +if config["sector"]["offshore_hubs_tyndp"]["enable"]: rule build_tyndp_offshore_hubs: params: planning_horizons=config_provider("scenario", "planning_horizons"), scenario=config_provider("tyndp_scenario"), countries=config_provider("countries"), + offshore_hubs_tyndp=config_provider("sector", "offshore_hubs_tyndp"), input: nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), @@ -1443,7 +1444,7 @@ rule prepare_sector_network: ), load_source=config_provider("load", "source"), scaling_factor=config_provider("load", "scaling_factor"), - offshore_hubs_tyndp=config_provider("sector", "offshore_hubs_tyndp"), + offshore_hubs_tyndp=config_provider("sector", "offshore_hubs_tyndp", "enable"), input: unpack(input_profile_offwind), unpack(input_profile_pecd), diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 5abb65df50..8c9b7f8867 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -15,7 +15,7 @@ rule solve_network: network=resources("networks/base_s_{clusters}_elec_{opts}.nc"), offshore_zone_trajectories=lambda w: ( resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs_tyndp")(w) + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) else [] ), output: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 8cbdc33019..29c1015044 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -93,7 +93,7 @@ rule add_brownfield: dynamic_ptes_capacity=config_provider( "sector", "district_heating", "ptes", "dynamic_capacity" ), - offshore_hubs_tyndp=config_provider("sector", "offshore_hubs_tyndp"), + offshore_hubs_tyndp=config_provider("sector", "offshore_hubs_tyndp", "enable"), input: unpack(input_profile_tech_brownfield), unpack(input_profile_tech_brownfied_pecd), @@ -144,7 +144,7 @@ rule solve_sector_network_myopic: costs=resources("costs_{planning_horizons}.csv"), offshore_zone_trajectories=lambda w: ( resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs_tyndp")(w) + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) else [] ), output: diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index e7fb103a2c..73364dcbb9 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -17,7 +17,7 @@ rule solve_sector_network: ), offshore_zone_trajectories=lambda w: ( resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs_tyndp")(w) + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) else [] ), output: diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 5b3f6a3c78..6d685cb7c5 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -109,7 +109,7 @@ rule solve_sector_network_perfect: costs=resources("costs_2030.csv"), offshore_zone_trajectories=lambda w: ( resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs_tyndp")(w) + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) else [] ), output: diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 58e3cde0c4..b84beb28ba 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -8,6 +8,7 @@ import logging import geopandas as gpd +import numpy as np import pandas as pd from _helpers import configure_logging, set_scenario_config from shapely.geometry import Point @@ -73,6 +74,7 @@ def load_offshore_grid( scenario: str, planning_horizons: list[int], countries: list[str], + max_capacity: dict[str, int], ): """ Load offshore grid (electricity and hydrogen) and format data. @@ -91,6 +93,8 @@ def load_offshore_grid( List of planning years to include in the cost data filtering. countries : list[str] List of country codes used to clean data. + max_capacity : dict[str, int] + Maximum transmission capacity between two offshore hubs per carrier Returns ------- @@ -153,6 +157,11 @@ def load_offshore_grid( grid[["capex", "opex"]] = grid[["capex", "opex"]].fillna(0) grid["p_nom_min"] = grid["p_nom_min"].fillna(0) + # Add maximum transmission capacities + grid["p_nom_max"] = np.where( + grid.carrier == "DC", max_capacity["DC"], max_capacity["H2"] + ) + # Rename UK in GB grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB", regex=True) @@ -518,6 +527,7 @@ def load_generators(sheet_name, tech_switch=None): snakemake.params["scenario"], planning_horizons, countries, + snakemake.params["offshore_hubs_tyndp"]["max_capacity"], ) electrolysers = load_offshore_electrolysers( diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index b7a711c0e7..53e864b9c3 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3279,6 +3279,7 @@ def add_offshore_grid_tyndp( p_nom_extendable=offshore_grid_dc.p_nom_extendable, p_nom=offshore_grid_dc.p_nom_min, p_nom_min=offshore_grid_dc.p_nom_min, + p_nom_max=offshore_grid_dc.p_nom_max, p_min_pu=offshore_grid_dc.p_min_pu, p_max_pu=offshore_grid_dc.p_max_pu, capital_cost=offshore_grid_dc.capital_cost, @@ -3313,6 +3314,7 @@ def add_offshore_grid_tyndp( p_nom_extendable=offshore_grid_h2.p_nom_extendable, p_nom=offshore_grid_h2.p_nom_min, p_nom_min=offshore_grid_h2.p_nom_min, + p_nom_max=offshore_grid_h2.p_nom_max, p_min_pu=offshore_grid_h2.p_min_pu, p_max_pu=offshore_grid_h2.p_max_pu, capital_cost=offshore_grid_h2.capital_cost, diff --git a/scripts/solve_network.py b/scripts/solve_network.py index bfe1eff6d8..27876efec1 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1298,7 +1298,7 @@ def extra_functionality( if config["sector"]["imports"]["enable"]: add_import_limit_constraint(n, snapshots) - if config["sector"]["offshore_hubs_tyndp"]: + if config["sector"]["offshore_hubs_tyndp"]["enable"]: add_offshore_hubs_constraint( n, int(planning_horizons), offshore_zone_trajectories_fn ) From 5c2de44774e57e36e48b2bd762b019d0947ff873 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 16:50:19 +0200 Subject: [PATCH 119/165] ci: fix run name keyword --- config/config.tyndp.yaml | 2 +- config/test/config.tyndp.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 03d4760eb7..a3db10e62b 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: CC0-1.0 run: - prefix: "tyndp" + name: "tyndp" # TODO set foresight foresight: myopic diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 22a71857e5..3b30342945 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: CC0-1.0 run: - prefix: "test-sector-tyndp" + name: "test-sector-tyndp" disable_progressbar: true shared_resources: policy: false From 225de715001262585dbeec57e419766d441d359e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20R=C3=BCdt?= <117752024+daniel-rdt@users.noreply.github.com> Date: Thu, 26 Jun 2025 17:22:16 +0200 Subject: [PATCH 120/165] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Gilon Co-authored-by: Jonas Hörsch --- doc/release_notes.rst | 2 +- rules/retrieve.smk | 2 +- rules/solve_myopic.smk | 4 ++-- scripts/add_electricity.py | 18 +++--------------- scripts/build_renewable_profiles_pecd.py | 13 ++++++------- scripts/clean_pecd_data.py | 5 ++--- scripts/retrieve_tyndp_pecd_data.py | 2 +- 7 files changed, 16 insertions(+), 30 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 7cb970c9df..be0d654ab4 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,7 +13,7 @@ Release Notes **Changes** -* Add processing and preparation of TYNDP 2024 PECD renewable profiles instead of default renewable profiles using atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. +* Add processing and preparation of TYNDP 2024 PECD renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards. diff --git a/rules/retrieve.smk b/rules/retrieve.smk index e23c6fa4bf..fcacc4428c 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -187,8 +187,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" rule retrieve_tyndp_pecd_data: params: + # TODO Integrate into Zenodo tyndp data bundle tyndp_bundle="data/tyndp_2024_bundle", - # TODO Integrate into Zenodo tyndp data bundle output: dir=directory("data/tyndp_2024_bundle/PECD"), log: diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index fb37bc164b..070a7e8cc2 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -69,7 +69,7 @@ def input_profile_tech_brownfield(w): } -def input_profile_tech_brownfied_pecd(w): +def input_profile_tech_brownfield_pecd(w): return { f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") for tech in pecd_renewable_profiles(w) @@ -94,7 +94,7 @@ rule add_brownfield: ), input: unpack(input_profile_tech_brownfield), - unpack(input_profile_tech_brownfied_pecd), + unpack(input_profile_tech_brownfield_pecd), simplify_busmap=resources("busmap_base_s.csv"), cluster_busmap=resources("busmap_base_s_{clusters}.csv"), network=resources( diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 1541df6e20..32b5649f3e 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1213,27 +1213,15 @@ def attach_stores( ) tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] + list(chain(*params.electricity["pecd_renewable_profiles"]["technologies"].values())) if params.electricity["pecd_renewable_profiles"]["enable"] else [] ) if len(tyndp_renewable_carriers) > 0: logger.info( - f"Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'. They will be attached later on with TYNDP data." + f"Skipping renewable carriers - they will be attached later with TYNDP data: {', '.join(tyndp_renewable_carriers)}" ) - renewable_carriers = set( - [ - carrier - for carrier in params.electricity["renewable_carriers"] - if carrier not in tyndp_renewable_carriers - ] - ) + renewable_carriers = set(params.electricity["renewable_carriers"]).difference(tyndp_renewable_carriers) extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 39197fb9c5..a7d102e1a2 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: MIT """ -Creates renewable profiles for each region from PECD and TYNDP input data containing the available -generation time series (based on PECD weather data) from the node for onshore wind, AC-connected offshore wind, -DC-connected offshore wind and solar PV generators. +Create renewable profiles for each region from PECD and TYNDP datasets, building on PECD climate data. The available +generation time series are read for each node across all renewable technologies, including onshore wind, AC-connected offshore wind, +DC-connected offshore wind, and solar PV generators. .. note:: Hydroelectric profiles will be built in script :mod:`build_hydro_profiles_PECD`. Not yet implemented. @@ -40,7 +40,7 @@ snakemake = mock_snakemake( "build_renewable_profiles_pecd", clusters="all", - technology="offwind-ac", + technology="offwind", ) configure_logging(snakemake) set_scenario_config(snakemake) @@ -58,12 +58,11 @@ if int(year) not in [2030, 2040, 2050]: year = np.clip(10 * (year // 10), 2030, 2050) logger.warning( - "Planning horizon doesn't match available TYNDP PECD data. " - f"Falling back to previous available year {year}." + f"TYNDP PECD data unavailable for planning horizon. Falling back to previous available year {year}." ) if year == 2050: logger.warning( - "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data instead." + "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data." ) year = 2040 diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 98f6826dec..ba30ecb3e1 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -37,7 +37,7 @@ def read_pecd_file( - node: list, + node: str, fn_pecd: str, cyear: str, pyear: str, @@ -140,8 +140,7 @@ def read_pecd_file( pecd_df = ( pd.concat(demand, axis=1) - .reindex(nodes, axis=1) - .fillna(0.0) # include missing node data with empty columns + .reindex(nodes, axis=1, fill_value=0.0) # include missing node data with empty columns .rename( columns=lambda x: x.replace("UK", "GB") ) # replace UK with GB for naming convention diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index a24bbc3675..67a5a84168 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -4,7 +4,7 @@ """ The TYNDP PECD data contains input data for the 2024 TYNDP scenario building process. -This rule downloads the TYNDP PECD data from `a gdrive and extracts it in the ``data/tyndp_2024_bundle`` +This rule downloads the TYNDP PECD data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` subdirectory, such that all files of the TYNDP bundle are stored in it. **Outputs** From faf71b5fbdf32a114b49da5334c1b073c556ce03 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 15:22:30 +0000 Subject: [PATCH 121/165] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 10 ++++++++-- scripts/clean_pecd_data.py | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 32b5649f3e..a53cfaeae8 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1213,7 +1213,11 @@ def attach_stores( ) tyndp_renewable_carriers = ( - list(chain(*params.electricity["pecd_renewable_profiles"]["technologies"].values())) + list( + chain( + *params.electricity["pecd_renewable_profiles"]["technologies"].values() + ) + ) if params.electricity["pecd_renewable_profiles"]["enable"] else [] ) @@ -1221,7 +1225,9 @@ def attach_stores( logger.info( f"Skipping renewable carriers - they will be attached later with TYNDP data: {', '.join(tyndp_renewable_carriers)}" ) - renewable_carriers = set(params.electricity["renewable_carriers"]).difference(tyndp_renewable_carriers) + renewable_carriers = set(params.electricity["renewable_carriers"]).difference( + tyndp_renewable_carriers + ) extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index ba30ecb3e1..9c2f19b599 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -140,7 +140,9 @@ def read_pecd_file( pecd_df = ( pd.concat(demand, axis=1) - .reindex(nodes, axis=1, fill_value=0.0) # include missing node data with empty columns + .reindex( + nodes, axis=1, fill_value=0.0 + ) # include missing node data with empty columns .rename( columns=lambda x: x.replace("UK", "GB") ) # replace UK with GB for naming convention From 81361210bbcc703f2dcc1f1a560e8a7713fa00c0 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 26 Jun 2025 17:41:19 +0200 Subject: [PATCH 122/165] fix: import chain from itertools --- scripts/add_electricity.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a53cfaeae8..a20c0c9677 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -50,6 +50,7 @@ import logging from collections.abc import Iterable +from itertools import chain from typing import Any import geopandas as gpd From a67fb078b38acf72e99a4eb946bf0a1c0c8b172c Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 26 Jun 2025 18:06:52 +0200 Subject: [PATCH 123/165] refactor: generalize get tyndp renewable carriers --- rules/solve_myopic.smk | 8 ++++++-- rules/solve_perfect.smk | 4 +++- scripts/add_brownfield.py | 14 +++---------- scripts/add_electricity.py | 34 ++++++++++++++++++++++++-------- scripts/add_existing_baseyear.py | 18 +++++++---------- 5 files changed, 45 insertions(+), 33 deletions(-) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 070a7e8cc2..e880b962ac 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -7,7 +7,9 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), - electricity=config_provider("electricity"), + pecd_renewable_profiles=config_provider( + "electricity", "pecd_renewable_profiles" + ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), @@ -84,7 +86,9 @@ rule add_brownfield: ), threshold_capacity=config_provider("existing_capacities", "threshold_capacity"), snapshots=config_provider("snapshots"), - electricity=config_provider("electricity"), + pecd_renewable_profiles=config_provider( + "electricity", "pecd_renewable_profiles" + ), drop_leap_day=config_provider("enable", "drop_leap_day"), carriers=config_provider("electricity", "renewable_carriers"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index ce9f905412..0b74cb35d1 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -5,7 +5,9 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), - electricity=config_provider("electricity"), + pecd_renewable_profiles=config_provider( + "electricity", "pecd_renewable_profiles" + ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 127c24d0bd..06e129ed56 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -19,7 +19,7 @@ set_scenario_config, update_config_from_wildcards, ) -from scripts.add_electricity import flatten, sanitize_carriers +from scripts.add_electricity import flatten, get_tyndp_res_carriers, sanitize_carriers from scripts.add_existing_baseyear import add_build_year_to_new_assets logger = logging.getLogger(__name__) @@ -356,16 +356,8 @@ def update_dynamic_ptes_capacity( n = pypsa.Network(snakemake.input.network) - tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] - if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] - else [] + tyndp_renewable_carriers = get_tyndp_res_carriers( + snakemake.params.pecd_renewable_profiles ) adjust_renewable_profiles( diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index a20c0c9677..38b5f9d3e3 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -190,6 +190,30 @@ def sanitize_carriers(n, config): n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors) +def get_tyndp_res_carriers(pecd_renewable_profiles: dict): + """ + Function to return all TYNDP renewable carriers specified in the configuration file for PECD profiles. + + The function makes sure TYNDP renewable carriers are only returned if PECD profiles are enabled. + + Parameters + ---------- + pecd_renewable_profiles : dict + Dictionary that contains all TYNDP renewable carrier for of the PECD profiles. + + Returns + ------- + tyndp_renewable_carriers : list + List of TYNDP renewable carriers. + """ + tyndp_renewable_carriers = ( + list(chain(*pecd_renewable_profiles["technologies"].values())) + if pecd_renewable_profiles["enable"] + else [] + ) + return tyndp_renewable_carriers + + def sanitize_locations(n): if "location" in n.buses.columns: n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x)) @@ -1213,14 +1237,8 @@ def attach_stores( params.link_length_factor, ) - tyndp_renewable_carriers = ( - list( - chain( - *params.electricity["pecd_renewable_profiles"]["technologies"].values() - ) - ) - if params.electricity["pecd_renewable_profiles"]["enable"] - else [] + tyndp_renewable_carriers = get_tyndp_res_carriers( + params.electricity["pecd_renewable_profiles"] ) if len(tyndp_renewable_carriers) > 0: logger.info( diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index c78d72bba8..3015db9635 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -23,7 +23,11 @@ set_scenario_config, update_config_from_wildcards, ) -from scripts.add_electricity import load_costs, sanitize_carriers +from scripts.add_electricity import ( + get_tyndp_res_carriers, + load_costs, + sanitize_carriers, +) from scripts.build_energy_totals import cartesian from scripts.definitions.heat_system import HeatSystem from scripts.prepare_sector_network import cluster_heat_buses, define_spatial @@ -749,16 +753,8 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector renewable_carriers = snakemake.params.carriers - tyndp_renewable_carriers = ( - [ - subcarrier - for carrier in snakemake.params.electricity["pecd_renewable_profiles"][ - "technologies" - ].values() - for subcarrier in carrier - ] - if snakemake.params.electricity["pecd_renewable_profiles"]["enable"] - else [] + tyndp_renewable_carriers = get_tyndp_res_carriers( + snakemake.params.pecd_renewable_profiles ) baseyear = snakemake.params.baseyear From 5313264c0ffb7e3ad2f9e4eaa96ba26cd7429e42 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 21:19:02 +0200 Subject: [PATCH 124/165] ci: fix run name keyword --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 3bdbc8eebc..fc977dd97b 100644 --- a/Snakefile +++ b/Snakefile @@ -79,7 +79,7 @@ if config["foresight"] == "perfect": rule all: input: expand(RESULTS + "graphs/costs.svg", run=config["run"]["name"]), - expand(resources("maps/power-network.pdf")), + expand(resources("maps/power-network.pdf"), run=config["run"]["name"]), expand( resources("maps/power-network-s-{clusters}.pdf"), run=config["run"]["name"], From 8da9d7d6990b4d964eb9cc997e7d8ae123819473 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 22:38:00 +0200 Subject: [PATCH 125/165] Revert "ci: fix run name keyword" This reverts commit 5c2de44774e57e36e48b2bd762b019d0947ff873. --- config/config.tyndp.yaml | 2 +- config/test/config.tyndp.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index a3db10e62b..03d4760eb7 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: CC0-1.0 run: - name: "tyndp" + prefix: "tyndp" # TODO set foresight foresight: myopic diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 3b30342945..22a71857e5 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: CC0-1.0 run: - name: "test-sector-tyndp" + prefix: "test-sector-tyndp" disable_progressbar: true shared_resources: policy: false From c77af19ffe7a64e9b2ba5be7053771ead950f8db Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 22:43:08 +0200 Subject: [PATCH 126/165] fix: fix merge conflict in add_brownfield regarding tyndp_renewable_carriers --- scripts/add_brownfield.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 288b68be6d..1abdd1c261 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -285,7 +285,9 @@ def disable_grid_expansion_if_limit_hit(n): n.global_constraints.drop(name, inplace=True) -def adjust_renewable_profiles(n, input_profiles, params, year): +def adjust_renewable_profiles( + n, input_profiles, params, year, tyndp_renewable_carriers +): """ Adjusts renewable profiles according to the renewable technology specified, using the latest year below or equal to the selected year. @@ -307,7 +309,7 @@ def adjust_renewable_profiles(n, input_profiles, params, year): } ) - for carrier in set(params["carriers"]): + for carrier in set(params["carriers"]) - set(tyndp_renewable_carriers): if carrier == "hydro": continue From c09708ac37a12b8c0dc293d8751b6a7affa34cdd Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 22:52:36 +0200 Subject: [PATCH 127/165] feat: make legend optional in plot_offshore_network --- scripts/plot_offshore_network.py | 115 +++++++++++++++++-------------- 1 file changed, 62 insertions(+), 53 deletions(-) diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index a488a3e61d..7cb54758e9 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -22,7 +22,13 @@ def plot_offshore_map( - network, map_opts, map_fn, planning_horizons, carrier="DC", expanded=False + network, + map_opts, + map_fn, + planning_horizons, + carrier="DC", + expanded=False, + legend=True, ): """ Plots the offshore network hydrogen and electricity capacities and offshore-hubs buses. @@ -42,6 +48,8 @@ def plot_offshore_map( Carrier to plot expanded : bool, optional Whether to plot expanded capacities. Defaults to plotting only base network (p_nom). + legend : bool, optional + Whether to display a legend on the plot. Defaults to display the legend. Returns ------- @@ -119,66 +127,67 @@ def plot_offshore_map( **map_opts, ) - sizes = [30, 10] - labels = [f"{s} GW" for s in sizes] - scale = 1e3 / linewidth_factor - sizes = [s * scale for s in sizes] - - legend_kw = dict( - loc="upper left", - bbox_to_anchor=(0.32, 1.13), - frameon=False, - ncol=1, - labelspacing=0.8, - handletextpad=1, - ) + if legend: + sizes = [30, 10] + labels = [f"{s} GW" for s in sizes] + scale = 1e3 / linewidth_factor + sizes = [s * scale for s in sizes] + + legend_kw = dict( + loc="upper left", + bbox_to_anchor=(0.32, 1.13), + frameon=False, + ncol=1, + labelspacing=0.8, + handletextpad=1, + ) - add_legend_lines( - ax, - sizes, - labels, - patch_kw=dict(color="lightgrey"), - legend_kw=legend_kw, - ) + add_legend_lines( + ax, + sizes, + labels, + patch_kw=dict(color="lightgrey"), + legend_kw=legend_kw, + ) - legend_kw = dict( - loc="upper left", - bbox_to_anchor=(0.55, 1.13), - labelspacing=0.8, - handletextpad=0, - frameon=False, - ) + legend_kw = dict( + loc="upper left", + bbox_to_anchor=(0.55, 1.13), + labelspacing=0.8, + handletextpad=0, + frameon=False, + ) - add_legend_circles( - ax, - sizes=[0.1], - labels=["Home market"], - srid=n.srid, - patch_kw=dict(facecolor=color_hm_nodes), - legend_kw=legend_kw, - ) + add_legend_circles( + ax, + sizes=[0.1], + labels=["Home market"], + srid=n.srid, + patch_kw=dict(facecolor=color_hm_nodes), + legend_kw=legend_kw, + ) - legend_kw["bbox_to_anchor"] = (0.55, 1.08) + legend_kw["bbox_to_anchor"] = (0.55, 1.08) - add_legend_circles( - ax, - sizes=[0.1], - labels=["Offshore hubs"], - srid=n.srid, - patch_kw=dict(facecolor=color_oh_nodes), - legend_kw=legend_kw, - ) + add_legend_circles( + ax, + sizes=[0.1], + labels=["Offshore hubs"], + srid=n.srid, + patch_kw=dict(facecolor=color_oh_nodes), + legend_kw=legend_kw, + ) - label = "DC link" if carrier == "DC" else "H2 pipeline" + label = "DC link" if carrier == "DC" else "H2 pipeline" - legend_kw = dict( - loc="upper left", - bbox_to_anchor=(0, 1.13), - ncol=1, - frameon=False, - ) + legend_kw = dict( + loc="upper left", + bbox_to_anchor=(0, 1.13), + ncol=1, + frameon=False, + ) - add_legend_patches(ax, color, label, legend_kw=legend_kw) + add_legend_patches(ax, color, label, legend_kw=legend_kw) ax.set_facecolor("white") From 1d51a96ae4f8586d44d55bb0fda365e517e5d27c Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 23:03:16 +0200 Subject: [PATCH 128/165] feat: modularize link width plotting functionality --- scripts/plot_offshore_network.py | 41 +++++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index 7cb54758e9..c7a4cbfe4a 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -27,7 +27,7 @@ def plot_offshore_map( map_fn, planning_horizons, carrier="DC", - expanded=False, + p_nom="p_nom", legend=True, ): """ @@ -46,8 +46,9 @@ def plot_offshore_map( The planning horizon year carrier : str, optional Carrier to plot - expanded : bool, optional - Whether to plot expanded capacities. Defaults to plotting only base network (p_nom). + p_nom : str | float, optional + Nominal power parameter for determining link thickness. If str, must be "p_nom" or "p_nom_opt". + If float, uses fixed value for all links. Defaults to plotting only base network (p_nom). legend : bool, optional Whether to display a legend on the plot. Defaults to display the legend. @@ -67,20 +68,24 @@ def plot_offshore_map( inplace=True, ) - p_nom = "p_nom_opt" if expanded else "p_nom" # transmission capacities - links = ( - n.links[n.links.index.str.contains(mask[carrier])][p_nom] - .rename(index=lambda x: re.sub(r"-\d{4}$", f"-{planning_horizons}", x)) - .groupby(level=0) - .sum() - ) - # set link widths - link_widths = links / linewidth_factor - if link_widths.notnull().empty: - logger.info(f"No offshore capacities for {carrier}, skipping plot.") - return - link_widths = link_widths.reindex(n.links.index).fillna(0.0) + if isinstance(p_nom, str): + links = ( + n.links[n.links.index.str.contains(mask[carrier])][p_nom] + .rename(index=lambda x: re.sub(r"-\d{4}$", f"-{planning_horizons}", x)) + .groupby(level=0) + .sum() + ) + # set link widths + link_widths = links / linewidth_factor + if link_widths.notnull().empty: + logger.info(f"No offshore capacities for {carrier}, skipping plot.") + return + link_widths = link_widths.reindex(n.links.index).fillna(0.0) + elif isinstance(p_nom, float) or isinstance(p_nom, int): + link_widths = p_nom + else: + raise ValueError("Value 'p_nom' must be either str or float.") # keep relevant buses bus_carriers = [carrier] + (["AC"] if carrier == "DC" else []) @@ -221,11 +226,13 @@ def plot_offshore_map( proj = load_projection(snakemake.params.plotting) map_fn = snakemake.output.map + p_nom = "p_nom_opt" if snakemake.params.expanded else "p_nom" + plot_offshore_map( n, map_opts, map_fn, snakemake.wildcards.planning_horizons, carrier=snakemake.wildcards.carrier, - expanded=snakemake.params.expanded, + p_nom=p_nom, ) From dac58ef510d30dac61dba729a2f4b31af993d25c Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 26 Jun 2025 23:31:00 +0200 Subject: [PATCH 129/165] feat: add hubs_only parameter to filter offshore hub in visualization --- scripts/plot_offshore_network.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index c7a4cbfe4a..e24f2a4984 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -29,6 +29,7 @@ def plot_offshore_map( carrier="DC", p_nom="p_nom", legend=True, + hubs_only=False, ): """ Plots the offshore network hydrogen and electricity capacities and offshore-hubs buses. @@ -51,6 +52,8 @@ def plot_offshore_map( If float, uses fixed value for all links. Defaults to plotting only base network (p_nom). legend : bool, optional Whether to display a legend on the plot. Defaults to display the legend. + hubs_only : bool, optional + Whether to only plot the offshore hubs. Defaults to plot both home market nodes and offshore hubs. Returns ------- @@ -102,6 +105,9 @@ def plot_offshore_map( n_oh.buses.index[~n_oh.buses.index.str.contains("OH")], inplace=True ) + if hubs_only: + n.buses = n_oh.buses + # plot transmission network logger.info("Plotting offshore transmission network.") proj = load_projection(dict(name="EqualEarth")) From b1931225b99223ee23d04fdc7ce25b77f8507743 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 13:35:55 +0200 Subject: [PATCH 130/165] fix: remove placeholder for renewable technology max capacities in build_renewable_profiles_pecd --- scripts/build_renewable_profiles_pecd.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index a7d102e1a2..2db9c11c2e 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -81,17 +81,4 @@ profiles.append(profile) ds = xr.merge(profiles) - - # TODO: Later on the max capacities for renewable technologies and regions can be added here - # profiles = xr.merge(profiles) - # p_nom_max = place_holder - # average_distance = place_holder - # ds = xr.merge( - # [ - # profiles, - # p_nom_max.rename("p_nom_max"), - # average_distance.rename("average_distance"), - # ] - # ) - ds.to_netcdf(snakemake.output.profile) From fed98dd08c6e07fb0586e6cf4bcd1b1579799f8f Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 13:47:36 +0200 Subject: [PATCH 131/165] doc: add information about PECD Version 3.1 to documentation --- doc/release_notes.rst | 2 +- scripts/retrieve_tyndp_pecd_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 66ff963092..2882f103f0 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,7 +13,7 @@ Release Notes **Changes** -* Add processing and preparation of TYNDP 2024 PECD renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. +* Add processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards. diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 67a5a84168..63d05d5d62 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -4,7 +4,7 @@ """ The TYNDP PECD data contains input data for the 2024 TYNDP scenario building process. -This rule downloads the TYNDP PECD data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` +This rule downloads the TYNDP PECD v3.1 data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` subdirectory, such that all files of the TYNDP bundle are stored in it. **Outputs** From 0a36daa061195315022e96b2f31e76bda2219605 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 14:00:30 +0200 Subject: [PATCH 132/165] fix: remove unnecessary params from pecd rules --- rules/build_electricity.smk | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 963535f5ed..8aa5b8b2ea 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -395,7 +395,6 @@ rule build_renewable_profiles: rule clean_pecd_data: params: - scenario=config_provider("tyndp_scenario"), snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), input: @@ -430,9 +429,6 @@ def input_data_pecd(w): rule build_renewable_profiles_pecd: params: - snapshots=config_provider("snapshots"), - drop_leap_day=config_provider("enable", "drop_leap_day"), - renewable=config_provider("renewable"), planning_horizons=config_provider("scenario", "planning_horizons"), input: unpack(input_data_pecd), From b98ecccff4bf16690bedc7df581a094bcde73d20 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 14:05:17 +0200 Subject: [PATCH 133/165] feat: change fn_pecd to dir_pecd and add comment on skiprows --- rules/build_electricity.smk | 2 +- scripts/clean_pecd_data.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 8aa5b8b2ea..4e1dcab164 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -400,7 +400,7 @@ rule clean_pecd_data: input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), - fn_pecd="data/tyndp_2024_bundle/PECD", + dir_pecd="data/tyndp_2024_bundle/PECD", output: pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), log: diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 9c2f19b599..0bfbaaae71 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -38,20 +38,20 @@ def read_pecd_file( node: str, - fn_pecd: str, + dir_pecd: str, cyear: str, pyear: str, technology: str, sns: pd.DatetimeIndex, ): - fn = Path(fn_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + fn = Path(dir_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") if not os.path.isfile(fn): return None pecd_bus = pd.read_csv( fn, - skiprows=10, + skiprows=10, # first ten rows contain only file metadata usecols=lambda name: name == "Date" or name == "Hour" or name == str(cyear) @@ -116,7 +116,7 @@ def read_pecd_file( nodes = ( offshore_buses.index if pecd_tech == "Wind_Offshore" else onshore_buses.index ) - fn_pecd = snakemake.input.fn_pecd + dir_pecd = snakemake.input.dir_pecd # Load and prep electricity demand tqdm_kwargs = { @@ -128,7 +128,7 @@ def read_pecd_file( func = partial( read_pecd_file, - fn_pecd=fn_pecd, + dir_pecd=dir_pecd, cyear=cyear, pyear=pyear, technology=pecd_tech, From 580eaaa025e64c45916215ca19adf9952af3d77a Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 14:16:28 +0200 Subject: [PATCH 134/165] refactor: change to if not continue style to reduce diff --- scripts/add_existing_baseyear.py | 81 ++++++++++++++++---------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 781708f1e7..0022b3e8af 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -113,46 +113,47 @@ def add_existing_renewables( irena = irena.unstack().reset_index() for carrier, tech in tech_map.items(): - if carrier in renewable_carriers: - df = ( - irena[irena.Technology.str.contains(tech)] - .drop(columns=["Technology"]) - .set_index("Country") - ) - df.columns = df.columns.astype(int) - - # calculate yearly differences - df.insert(loc=0, value=0.0, column="1999") - df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0) - - # distribute capacities among generators potential (p_nom_max) - gen_i = n.generators.query("carrier == @carrier").index - carrier_gens = n.generators.loc[gen_i] - res_capacities = [] - for country, group in carrier_gens.groupby( - carrier_gens.bus.map(n.buses.country) - ): - fraction = group.p_nom_max / group.p_nom_max.sum() - res_capacities.append(cartesian(df.loc[country], fraction)) - res_capacities = pd.concat(res_capacities, axis=1).T - - for year in res_capacities.columns: - for gen in res_capacities.index: - bus_bin = re.sub(f" {carrier}.*", "", gen) - bus, bin_id = bus_bin.rsplit(" ", maxsplit=1) - name = f"{bus_bin} {carrier}-{year}" - capacity = res_capacities.loc[gen, year] - if capacity > 0.0: - cost_key = carrier.split("-", maxsplit=1)[0] - df_agg.at[name, "Fueltype"] = carrier - df_agg.at[name, "Capacity"] = capacity - df_agg.at[name, "DateIn"] = year - df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"] - df_agg.at[name, "DateOut"] = ( - year + costs.at[cost_key, "lifetime"] - 1 - ) - df_agg.at[name, "bus"] = bus - df_agg.at[name, "resource_class"] = bin_id + if carrier not in renewable_carriers: + continue + df = ( + irena[irena.Technology.str.contains(tech)] + .drop(columns=["Technology"]) + .set_index("Country") + ) + df.columns = df.columns.astype(int) + + # calculate yearly differences + df.insert(loc=0, value=0.0, column="1999") + df = df.diff(axis=1).drop("1999", axis=1).clip(lower=0) + + # distribute capacities among generators potential (p_nom_max) + gen_i = n.generators.query("carrier == @carrier").index + carrier_gens = n.generators.loc[gen_i] + res_capacities = [] + for country, group in carrier_gens.groupby( + carrier_gens.bus.map(n.buses.country) + ): + fraction = group.p_nom_max / group.p_nom_max.sum() + res_capacities.append(cartesian(df.loc[country], fraction)) + res_capacities = pd.concat(res_capacities, axis=1).T + + for year in res_capacities.columns: + for gen in res_capacities.index: + bus_bin = re.sub(f" {carrier}.*", "", gen) + bus, bin_id = bus_bin.rsplit(" ", maxsplit=1) + name = f"{bus_bin} {carrier}-{year}" + capacity = res_capacities.loc[gen, year] + if capacity > 0.0: + cost_key = carrier.split("-", maxsplit=1)[0] + df_agg.at[name, "Fueltype"] = carrier + df_agg.at[name, "Capacity"] = capacity + df_agg.at[name, "DateIn"] = year + df_agg.at[name, "lifetime"] = costs.at[cost_key, "lifetime"] + df_agg.at[name, "DateOut"] = ( + year + costs.at[cost_key, "lifetime"] - 1 + ) + df_agg.at[name, "bus"] = bus + df_agg.at[name, "resource_class"] = bin_id df_agg["resource_class"] = df_agg["resource_class"].fillna(0) From a55cfbf1ef8bafb2ba651be68d6617e716984c93 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 14:44:31 +0200 Subject: [PATCH 135/165] feat: improve pecd profile configuration by making profile names explicit and improving documentation --- config/config.default.yaml | 2 +- config/config.tyndp.yaml | 2 +- config/test/config.tyndp.yaml | 2 +- doc/configtables/electricity.csv | 4 ++-- scripts/clean_pecd_data.py | 13 ++----------- 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 038de075d8..d8ddb03f3a 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -116,7 +116,7 @@ electricity: pecd_renewable_profiles: enable: false technologies: - offwind: + Wind_Offshore: - offwind-ac-fb-r - offwind-ac-fl-r - offwind-dc-fb-r diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index af2822fd44..1d94f1349d 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -43,7 +43,7 @@ electricity: pecd_renewable_profiles: enable: true technologies: - offwind: + Wind_Offshore: - offwind-ac-fb-r - offwind-ac-fl-r - offwind-dc-fb-r diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index eab5f431c2..beed2b657d 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -41,7 +41,7 @@ electricity: pecd_renewable_profiles: enable: true technologies: - offwind: + Wind_Offshore: - offwind-ac-fb-r - offwind-ac-fl-r - offwind-dc-fb-r diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 3af8782456..bbd5199c05 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -31,9 +31,9 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite ,,, renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. pecd_renewable_profiles,,, --- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_profile_name`` can be any of {Wind_Offshore, Wind_Onshore, LFSolarPV}. -- technologies,,, --- -- {pecd_profile},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. Carriers not listed use default renewable profiles. +-- -- {pecd_profile_name},--,list,The TYNDP renewable carriers for which the PECD renewable profile is used. These TYNDP renewable carriers should also be listed in ``renewable_carriers``. Carriers not listed use default renewable profiles. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 0bfbaaae71..ebe8da9c38 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -97,18 +97,9 @@ def read_pecd_file( # Planning year pyear = str(snakemake.wildcards.planning_horizons) - # Technology as in PECD terminology # TODO: find solution for solar profiles being differentiated between Utility and Rooftop for Italy - pecd_tech_dict = { - "offwind-ac": "Wind_Offshore", - "offwind-dc": "Wind_Offshore", - "offwind-float": "Wind_Offshore", - "offwind": "Wind_Offshore", - "onwind": "Wind_Onshore", - "solar": "LFSolarPV", - "solar-hsat": "LFSolarPV", - } - pecd_tech = pecd_tech_dict[snakemake.wildcards.technology] + # Technology as in PECD terminology + pecd_tech = snakemake.wildcards.technology offshore_buses = pd.read_excel(snakemake.input.offshore_buses, index_col=0) onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) From decd1868b02efd90b576793a36f997558a3216ad Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 27 Jun 2025 15:19:45 +0200 Subject: [PATCH 136/165] refactor: simplify tyndp_renewable_carriers also in smk files --- Snakefile | 1 + rules/build_electricity.smk | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Snakefile b/Snakefile index f21e0c5568..a6c449205a 100644 --- a/Snakefile +++ b/Snakefile @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: MIT +from itertools import chain from pathlib import Path import yaml from os.path import normpath, exists, join diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 4e1dcab164..455a7762af 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -788,13 +788,13 @@ rule cluster_network: def tyndp_renewable_carriers(w): return ( - [ - subcarrier - for carrier in config_provider( - "electricity", "pecd_renewable_profiles", "technologies" - )(w).values() - for subcarrier in carrier - ] + list( + chain( + *config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w).values() + ) + ) if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) else [] ) From 55e9600b1945c8fa5fec5776ae27a9141c64a292 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 30 Jun 2025 10:34:52 +0200 Subject: [PATCH 137/165] feat: improve mapping from tyndp techs to PECD profile names --- scripts/prepare_sector_network.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 53e864b9c3..aec8963a72 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3036,6 +3036,7 @@ def add_offshore_generators_tyndp( pyear: int, offshore_generators_fn: str, profiles: dict[str, str], + pecd_mapping: dict[str, str], costs: pd.DataFrame, logger: logging.Logger, nyears: float = 1, @@ -3060,6 +3061,9 @@ def add_offshore_generators_tyndp( profiles : dict[str, str] Dictionary mapping technology names to profile file paths e.g. {'offwind-dc': 'path/to/profile.nc'} + pecd_mapping : dict[str, str] + Dictionary mapping technology names to PECD profile names + e.g. {'offwind-dc-fb-oh': 'Offshore_Wind'} costs : pd.DataFrame Technology costs assumptions. logger : logging.Logger @@ -3101,12 +3105,17 @@ def add_offshore_generators_tyndp( + offshore_generators["opex"] ) * nyears + # mapping from TYNDP offshore generators to PECD profiles + offshore_generators["pecd_profile_name"] = offshore_generators["carrier"].map( + pecd_mapping + ) + # Load PECD profiles p_max_pu = [] for key, fn in profiles.items(): tech = key[len("profile_pecd_") :] techs = offshore_generators[ - offshore_generators.carrier.str.contains(tech) + offshore_generators.pecd_profile_name == tech ].carrier.unique() with xr.open_dataset(fn) as ds: @@ -3330,6 +3339,7 @@ def add_offshore_hubs_tyndp( offshore_electrolysers_fn: str, offshore_grid_fn: str, profiles: dict[str, str], + pecd_mapping: dict[str, str], costs: pd.DataFrame, spatial: SimpleNamespace, logger: logging.Logger, @@ -3356,6 +3366,9 @@ def add_offshore_hubs_tyndp( profiles : dict[str, str] Dictionary mapping technology names to profile file paths e.g. {'offwind-dc': 'path/to/profile.nc'} + pecd_mapping : dict[str, str] + Dictionary mapping technology names to PECD profile names + e.g. {'offwind-dc-fb-oh': 'Offshore_Wind'} costs : pd.DataFrame Technology costs assumptions. spatial : object, optional @@ -3405,7 +3418,7 @@ def add_offshore_hubs_tyndp( # Add power production units add_offshore_generators_tyndp( - n, pyear, offshore_generators_fn, profiles, costs, logger, nyears + n, pyear, offshore_generators_fn, profiles, pecd_mapping, costs, logger, nyears ) # Add H2 production units @@ -7550,6 +7563,13 @@ def add_import_options( for key in snakemake.input.keys() if key.startswith("profile") } + pecd_renewable_profiles_techs = snakemake.params.electricity[ + "pecd_renewable_profiles" + ]["technologies"] + pecd_mapping = { + v: k for k, v_list in pecd_renewable_profiles_techs.items() for v in v_list + } + landfall_lengths = { tech: settings["landfall_length"] for tech, settings in snakemake.params.renewable.items() @@ -7628,6 +7648,7 @@ def add_import_options( offshore_electrolysers_fn=snakemake.input.offshore_electrolysers, offshore_grid_fn=snakemake.input.offshore_grid, profiles=profiles, + pecd_mapping=pecd_mapping, costs=costs, spatial=spatial, logger=logger, From 38266f6d12d00706148b5386c79877ed18f5f049 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 4 Jul 2025 07:38:31 +0200 Subject: [PATCH 138/165] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniel Rüdt <117752024+daniel-rdt@users.noreply.github.com> --- doc/configtables/sector.csv | 2 +- doc/release_notes.rst | 2 +- scripts/add_brownfield.py | 6 ++++-- scripts/plot_offshore_network.py | 8 ++++---- scripts/prepare_sector_network.py | 2 +- scripts/solve_network.py | 4 ++-- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/doc/configtables/sector.csv b/doc/configtables/sector.csv index 3057b1183f..956965219f 100644 --- a/doc/configtables/sector.csv +++ b/doc/configtables/sector.csv @@ -232,5 +232,5 @@ imports,,, -- -- {carrier},currency/MWh,float,Price for importing renewable energy of carrier offshore_hubs_tyndp,--,"{true, false}",Add options for TYNDP offshore hubs -- enable,--,"{true, false}",Add option to include TYNDP offshore hubs --- max_capacity,--,"{true, false}",Maximum transmission capacities between two offshore hubs +-- max_capacity,,, -- -- {carrier},GW,float,Maximum transmission capacity between two offshore hubs of a carrier \ No newline at end of file diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 909b709b4a..f35f204b2f 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,7 +13,7 @@ Release Notes **Changes** -* Introduce TYNDP offshore wind hubs via `sector:offshore_hubs_tyndp` configuration (https://github.com/open-energy-transition/open-tyndp/pull/54). This feature implements an offshore grid topology with both electric and hydrogen infrastructure, offshore electrolysers, and detailed wind farm characteristics. Three wind farm types are supported: AC-radial, DC-radial and DC-hubs. Each is compatible with fixed-bottom or floating foundations. Wind farms connected to hubs can produce hydrogen directly through a dedicated P2G unit, while electricity can be supplied to the network or converted to hydrogen via offshore electrolysers connected to the hubs. The network includes existing capacities, with capacity expansion constrained by technological potential and evolving zone potential. +* Introduce TYNDP offshore wind hubs via `sector:offshore_hubs_tyndp` configuration (https://github.com/open-energy-transition/open-tyndp/pull/54). This feature implements an offshore grid topology with both electric and hydrogen infrastructure, offshore electrolysers, and detailed wind farm characteristics. Three wind farm types are supported: AC-radial (ac-r), DC-radial (dc-r) and DC-hubs (dc-oh). Each is compatible with fixed-bottom (fb) or floating (fl) foundations. Wind farms connected to hubs can produce hydrogen directly through a dedicated P2G unit, while electricity can be supplied to the network or converted to hydrogen via offshore electrolysers connected to the hubs. The network includes existing capacities, with capacity expansion constrained by technological potential and evolving zone potential. * Add processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1abdd1c261..17fd43116b 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -171,11 +171,13 @@ def add_brownfield( index=lambda x: x.replace("dc", "h2") ) - already_existing = ( + already_existing_l = ( pd.concat([already_existing, h2_to_dc, dc_to_h2]) .groupby(level=0) .sum() ) + else: + already_existing_l = already_existing.copy() # values should be non-negative; clipping applied to handle rounding errors remaining_capacity = ( @@ -184,7 +186,7 @@ def add_brownfield( ).clip(lower=0) remaining_potential = ( off_potential - - already_existing.reindex(index=off_capacity.index).fillna(0) + - already_existing_l.reindex(index=off_capacity.index).fillna(0) ).clip(lower=0) c.df.loc[off_i, ["p_nom_min", "p_nom"]] = remaining_capacity c.df.loc[off_i, "p_nom_max"] = remaining_potential diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index e24f2a4984..53551cb046 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: MIT """ -Plot offshore transmission network. +Plot offshore transmission network with existing capacities. If `expanded` is enabled, the optimal capacities are plotted instead. """ import logging @@ -33,7 +33,7 @@ def plot_offshore_map( ): """ Plots the offshore network hydrogen and electricity capacities and offshore-hubs buses. - If expanded is enabled, the optimal capacities are plotted instead. + If `p_nom` parameter is set as `p_nom_opt`, optimal capacities are plotted instead. Parameters ---------- @@ -58,7 +58,7 @@ def plot_offshore_map( Returns ------- None - Saves the map plot as figure. + Saves the map plot as figure to the provided map_fn path. """ n = network.copy() @@ -88,7 +88,7 @@ def plot_offshore_map( elif isinstance(p_nom, float) or isinstance(p_nom, int): link_widths = p_nom else: - raise ValueError("Value 'p_nom' must be either str or float.") + raise ValueError("Parameter 'p_nom' must be either str or float.") # keep relevant buses bus_carriers = [carrier] + (["AC"] if carrier == "DC" else []) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index aec8963a72..4b2b8c2e8f 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3056,7 +3056,7 @@ def add_offshore_generators_tyndp( The network object to add offshore generators to. pyear : int Planning horizon used to filter which reference generator data to include. - offshore_generators : str + offshore_generators_fn : str Path to the file containing offshore generators configuration data. profiles : dict[str, str] Dictionary mapping technology names to profile file paths diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 27876efec1..6ff61fe35f 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1137,7 +1137,7 @@ def add_offshore_hubs_constraint( planning_horizons : int, optional The current planning horizon year or None in perfect foresight offshore_zone_trajectories_fn: str - Path to the dataFrame containing the offshore zone potentials trajectories + Path to the file containing the offshore zone potentials trajectories """ ext_i = n.generators.p_nom_extendable gens = n.generators.assign( @@ -1241,7 +1241,7 @@ def extra_functionality( planning_horizons : str, optional The current planning horizon year or None in perfect foresight offshore_zone_trajectories_fn: str, optional - Path to the DataFrame containing the offshore zone potentials trajectories + Path to the file containing the offshore zone potentials trajectories Collects supplementary constraints which will be passed to ``pypsa.optimization.optimize``. From 93f371ba6a93bc66bbad48619c8c45d147f860aa Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 7 Jul 2025 11:18:51 +0200 Subject: [PATCH 139/165] fix: fix merge conflicts --- rules/build_electricity.smk | 15 --------------- rules/build_sector.smk | 21 --------------------- rules/solve_myopic.smk | 13 ------------- rules/solve_perfect.smk | 6 ------ scripts/add_brownfield.py | 26 ++++---------------------- scripts/add_electricity.py | 25 ------------------------- scripts/add_existing_baseyear.py | 15 --------------- 7 files changed, 4 insertions(+), 117 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 4f604216fa..0797d3aef8 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -601,7 +601,6 @@ def input_class_regions(w): ) for tech in set(config_provider("electricity", "renewable_carriers")(w)) - {"hydro"} - - set(tyndp_renewable_carriers(w)) } @@ -786,20 +785,6 @@ rule cluster_network: "../scripts/cluster_network.py" -def tyndp_renewable_carriers(w): - return ( - list( - chain( - *config_provider( - "electricity", "pecd_renewable_profiles", "technologies" - )(w).values() - ) - ) - if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) - else [] - ) - - def input_profile_tech(w): return { f"profile_{tech}": resources( diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 6f6e5195dd..5696dcdcd5 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1228,27 +1228,6 @@ def input_profile_offwind(w): f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") for tech in ["offwind-ac", "offwind-dc", "offwind-float"] if (tech in config_provider("electricity", "renewable_carriers")(w)) - and (tech not in tyndp_renewable_carriers(w)) - } - - -def pecd_renewable_profiles(w): - return ( - [ - carrier - for carrier in config_provider( - "electricity", "pecd_renewable_profiles", "technologies" - )(w).keys() - ] - if config_provider("electricity", "pecd_renewable_profiles", "enable")(w) - else [] - ) - - -def input_profile_pecd(w): - return { - f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") - for tech in pecd_renewable_profiles(w) } diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 48d7eb74c6..ea7daffa83 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -7,12 +7,6 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), - pecd_renewable_profiles=config_provider( - "electricity", "pecd_renewable_profiles" - ), - tyndp_renewable_carriers=config_provider( - "electricity", "tyndp_renewable_carriers" - ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), @@ -86,15 +80,8 @@ rule add_brownfield: ), threshold_capacity=config_provider("existing_capacities", "threshold_capacity"), snapshots=config_provider("snapshots"), - pecd_renewable_profiles=config_provider( - "electricity", "pecd_renewable_profiles" - ), - tyndp_renewable_carriers=config_provider( - "electricity", "tyndp_renewable_carriers" - ), drop_leap_day=config_provider("enable", "drop_leap_day"), carriers=config_provider("electricity", "renewable_carriers"), - carriers_pecd=config_provider("electricity", "pecd_renewable_profiles"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), tes=config_provider("sector", "tes"), dynamic_ptes_capacity=config_provider( diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 8d77ab520d..28b2aff166 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -5,12 +5,6 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), - pecd_renewable_profiles=config_provider( - "electricity", "pecd_renewable_profiles" - ), - tyndp_renewable_carriers=config_provider( - "electricity", "tyndp_renewable_carriers" - ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index fdec48433e..6566417093 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -287,9 +287,7 @@ def disable_grid_expansion_if_limit_hit(n): n.global_constraints.drop(name, inplace=True) -def adjust_renewable_profiles( - n, input_profiles, params, year, tyndp_renewable_carriers -): +def adjust_renewable_profiles(n, input_profiles, params, year): """ Adjusts renewable profiles according to the renewable technology specified, using the latest year below or equal to the selected year. @@ -301,23 +299,11 @@ def adjust_renewable_profiles( pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill() ) - fn_map = {i: i for i in params["carriers"]} - if params["carriers_pecd"].get("enable", False): - fn_map.update( - { - vi: k - for k, v in params["carriers_pecd"]["technologies"].items() - for vi in v - } - ) - - for carrier in set(params["carriers"]) - set(tyndp_renewable_carriers): + for carrier in set(params["carriers"]): if carrier == "hydro": continue - with xr.open_dataset( - getattr(input_profiles, "profile_" + fn_map[carrier]) - ) as ds: + with xr.open_dataset(getattr(input_profiles, "profile_" + carrier)) as ds: if ds.indexes["bus"].empty or "year" not in ds.indexes: continue @@ -445,11 +431,7 @@ def update_dynamic_ptes_capacity( n = pypsa.Network(snakemake.input.network) - tyndp_renewable_carriers = snakemake.params.tyndp_renewable_carriers - - adjust_renewable_profiles( - n, snakemake.input, snakemake.params, year, tyndp_renewable_carriers - ) + adjust_renewable_profiles(n, snakemake.input, snakemake.params, year) add_build_year_to_new_assets(n, year) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 5e64d23309..a52a72dd58 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -50,7 +50,6 @@ import logging from collections.abc import Iterable -from itertools import chain from typing import Any import geopandas as gpd @@ -191,30 +190,6 @@ def sanitize_carriers(n, config): n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors) -def get_tyndp_res_carriers(pecd_renewable_profiles: dict): - """ - Function to return all TYNDP renewable carriers specified in the configuration file for PECD profiles. - - The function makes sure TYNDP renewable carriers are only returned if PECD profiles are enabled. - - Parameters - ---------- - pecd_renewable_profiles : dict - Dictionary that contains all TYNDP renewable carrier for of the PECD profiles. - - Returns - ------- - tyndp_renewable_carriers : list - List of TYNDP renewable carriers. - """ - tyndp_renewable_carriers = ( - list(chain(*pecd_renewable_profiles["technologies"].values())) - if pecd_renewable_profiles["enable"] - else [] - ) - return tyndp_renewable_carriers - - def sanitize_locations(n): if "location" in n.buses.columns: n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x)) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 1eae4a215d..5c08ba485e 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -72,7 +72,6 @@ def add_existing_renewables( df_agg: pd.DataFrame, countries: list[str], renewable_carriers: list[str], - tyndp_renewable_carriers: list[str], ) -> None: """ Add existing renewable capacities to conventional power plant data. @@ -89,8 +88,6 @@ def add_existing_renewables( List of country codes to consider renewable_carriers: list List of renewable carriers in the network - tyndp_renewable_carriers: list - List of renewable technologies from TYNDP Returns ------- @@ -98,12 +95,6 @@ def add_existing_renewables( Modifies df_agg in-place """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} - # TODO: remove when TYNDP renewable generators are added - if len(tyndp_renewable_carriers) > 0: - logger.info( - f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." - ) - renewable_carriers = set(renewable_carriers) - set(tyndp_renewable_carriers) irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() irena = irena.query("Country in @countries") @@ -167,7 +158,6 @@ def add_power_capacities_installed_before_baseyear( capacity_threshold: float, lifetime_values: dict[str, float], renewable_carriers: list[str], - tyndp_renewable_carriers: list[str], ) -> None: """ Add power generation capacities installed before base year. @@ -192,8 +182,6 @@ def add_power_capacities_installed_before_baseyear( Default values for missing data renewable_carriers: list List of renewable carriers in the network - tyndp_renewable_carriers: list - List of renewable technologies from TYNDP """ logger.debug(f"Adding power capacities installed before {baseyear}") @@ -247,7 +235,6 @@ def add_power_capacities_installed_before_baseyear( n=n, countries=countries, renewable_carriers=renewable_carriers, - tyndp_renewable_carriers=tyndp_renewable_carriers, ) # drop assets which are already phased out / decommissioned phased_out = df_agg[df_agg["DateOut"] < baseyear].index @@ -760,7 +747,6 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector renewable_carriers = snakemake.params.carriers - tyndp_renewable_carriers = snakemake.params.tyndp_renewable_carriers baseyear = snakemake.params.baseyear @@ -789,7 +775,6 @@ def add_heating_capacities_installed_before_baseyear( capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"], lifetime_values=snakemake.params.costs["fill_values"], renewable_carriers=renewable_carriers, - tyndp_renewable_carriers=tyndp_renewable_carriers, ) if options["heating"]: From 1fd73789408ee90653d20faf57bd8349960019a6 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 2 Jul 2025 09:43:13 +0200 Subject: [PATCH 140/165] refactor: remove logger as argument --- scripts/prepare_sector_network.py | 56 ++++++------------------------- 1 file changed, 10 insertions(+), 46 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 1f94a3aade..aaeb390bf2 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -2359,7 +2359,7 @@ def add_h2_topology_tyndp( ) -def add_h2_production(n, nodes, options, spatial, costs, logger): +def add_h2_production(n, nodes, options, spatial, costs): """ Adds base H2 production technologies. @@ -2379,8 +2379,6 @@ def add_h2_production(n, nodes, options, spatial, costs, logger): Object containing spatial information about nodes and their locations costs : pd.DataFrame Technology cost assumptions - logger : logging.Logger, optional - Logger for output messages. If None, no logging is performed. Returns ------- @@ -2437,7 +2435,7 @@ def add_h2_production(n, nodes, options, spatial, costs, logger): ) -def add_h2_reconversion(n, nodes, options, spatial, costs, logger): +def add_h2_reconversion(n, nodes, options, spatial, costs): """ Adds base H2 reconversion technologies (optional). @@ -2457,8 +2455,6 @@ def add_h2_reconversion(n, nodes, options, spatial, costs, logger): Object containing spatial information about nodes and their locations costs : pd.DataFrame Technology cost assumptions - logger : logging.Logger, optional - Logger for output messages. If None, no logging is performed. Returns ------- @@ -2521,7 +2517,7 @@ def add_h2_reconversion(n, nodes, options, spatial, costs, logger): ) -def add_h2_storage(n, nodes, options, cavern_types, h2_cavern_file, costs, logger): +def add_h2_storage(n, nodes, options, cavern_types, h2_cavern_file, costs): """ Adds H2 storage as underground cavern storage (optional) and H2 steel tanks. @@ -2541,8 +2537,6 @@ def add_h2_storage(n, nodes, options, cavern_types, h2_cavern_file, costs, logge Path to CSV file containing hydrogen cavern storage potentials costs : pd.DataFrame Technology cost assumptions - logger : logging.Logger, optional - Logger for output messages. If None, no logging is performed. Returns ------- @@ -2602,7 +2596,7 @@ def add_h2_storage(n, nodes, options, cavern_types, h2_cavern_file, costs, logge ) -def add_gas_network(n, gas_pipes, options, costs, gas_input_nodes, logger): +def add_gas_network(n, gas_pipes, options, costs, gas_input_nodes): """ Adds natural gas infrastructure, incl. LNG terminals, production, storage and entry-points. @@ -2621,8 +2615,6 @@ def add_gas_network(n, gas_pipes, options, costs, gas_input_nodes, logger): Technology cost assumptions gas_input_nodes : pd.DataFrame, optional DataFrame containing gas input node information (LNG, pipeline, etc.) - logger : logging.Logger, optional - Logger for output messages. If None, no logging is performed. Returns ------- @@ -2736,7 +2728,7 @@ def add_gas_network(n, gas_pipes, options, costs, gas_input_nodes, logger): ) -def add_h2_pipeline_retrofit(n, gas_pipes, options, costs, logger): +def add_h2_pipeline_retrofit(n, gas_pipes, options, costs): """ Adds retrofitting options of existing CH4 pipes to H2 pipes. @@ -2752,8 +2744,6 @@ def add_h2_pipeline_retrofit(n, gas_pipes, options, costs, logger): - H2_retrofit_capacity_per_CH4 : float costs : pd.DataFrame Technology cost assumptions - logger : logging.Logger, optional - Logger for output messages. If None, no logging is performed. Returns ------- @@ -2784,7 +2774,7 @@ def add_h2_pipeline_retrofit(n, gas_pipes, options, costs, logger): ) -def add_h2_pipeline_new(n, costs, logger): +def add_h2_pipeline_new(n, costs): """ Adds options for new H2 pipelines. @@ -2794,8 +2784,6 @@ def add_h2_pipeline_new(n, costs, logger): The PyPSA network container object costs : pd.DataFrame Technology cost assumptions - logger : logging.Logger, optional - Logger for output messages. If None, no logging is performed. Returns ------- @@ -2895,7 +2883,6 @@ def add_gas_and_h2_infrastructure( gas_input_nodes, spatial, options, - logger, ): """ Add storage and grid infrastructure to the network for gas and hydrogen. @@ -2935,8 +2922,6 @@ def add_gas_and_h2_infrastructure( - SMR : bool - cc_fraction : float - methanation : bool - logger : logging.Logger, optional - Logger for output messages. If None, no logging is performed. Returns ------- @@ -2985,7 +2970,6 @@ def add_gas_and_h2_infrastructure( options=options, spatial=spatial, costs=costs, - logger=logger, ) add_h2_reconversion( n=n, @@ -2993,7 +2977,6 @@ def add_gas_and_h2_infrastructure( options=options, spatial=spatial, costs=costs, - logger=logger, ) add_h2_storage( n=n, @@ -3002,7 +2985,6 @@ def add_gas_and_h2_infrastructure( cavern_types=cavern_types, h2_cavern_file=h2_cavern_file, costs=costs, - logger=logger, ) # add gas network, along with new and retrofitted H2 pipelines @@ -3016,7 +2998,6 @@ def add_gas_and_h2_infrastructure( options=options, costs=costs, gas_input_nodes=gas_input_nodes, - logger=logger, ) if options["H2_retrofit"] and not options["h2_topology_tyndp"]: @@ -3025,11 +3006,10 @@ def add_gas_and_h2_infrastructure( gas_pipes=gas_pipes, options=options, costs=costs, - logger=logger, ) if options["H2_network"] and not options["h2_topology_tyndp"]: - add_h2_pipeline_new(n=n, costs=costs, logger=logger) + add_h2_pipeline_new(n=n, costs=costs) def add_offshore_generators_tyndp( @@ -3039,7 +3019,6 @@ def add_offshore_generators_tyndp( profiles: dict[str, str], pecd_mapping: dict[str, str], costs: pd.DataFrame, - logger: logging.Logger, nyears: float = 1, ): """ @@ -3067,8 +3046,6 @@ def add_offshore_generators_tyndp( e.g. {'offwind-dc-fb-oh': 'Offshore_Wind'} costs : pd.DataFrame Technology costs assumptions. - logger : logging.Logger - Logger for output messages. If None, no logging is performed. nyears : float, default 1 Number of years for which to scale the investment costs. @@ -3153,7 +3130,6 @@ def add_offshore_electrolysers_tyndp( pyear: int, offshore_electrolysers_fn: str, costs: pd.DataFrame, - logger: logging.Logger, nyears: float = 1, ): """ @@ -3171,8 +3147,6 @@ def add_offshore_electrolysers_tyndp( Path to the file containing offshore electrolysers configuration data. costs : pd.DataFrame Technology costs assumptions. - logger : logging.Logger - Logger for output messages. If None, no logging is performed. nyears : float, default 1 Number of years for which to scale the investment costs. @@ -3231,7 +3205,6 @@ def add_offshore_grid_tyndp( pyear: int, offshore_grid_fn: str, costs: pd.DataFrame, - logger: logging.Logger, nyears: float = 1, ): """ @@ -3249,8 +3222,6 @@ def add_offshore_grid_tyndp( Path to the file containing offshore grid configuration data. costs : pd.DataFrame Technology costs assumptions. - logger : logging.Logger - Logger for output messages. If None, no logging is performed. nyears : float, default 1 Number of years for which to scale the investment costs. @@ -3343,7 +3314,6 @@ def add_offshore_hubs_tyndp( pecd_mapping: dict[str, str], costs: pd.DataFrame, spatial: SimpleNamespace, - logger: logging.Logger, nyears: float = 1, ): """ @@ -3374,8 +3344,6 @@ def add_offshore_hubs_tyndp( Technology costs assumptions. spatial : object, optional Object containing spatial information about nodes and their locations. - logger : logging.Logger, optional - Logger for output messages. If None, no logging is performed. nyears : float Number of years for which to scale the investment costs. @@ -3419,16 +3387,14 @@ def add_offshore_hubs_tyndp( # Add power production units add_offshore_generators_tyndp( - n, pyear, offshore_generators_fn, profiles, pecd_mapping, costs, logger, nyears + n, pyear, offshore_generators_fn, profiles, pecd_mapping, costs, nyears ) # Add H2 production units - add_offshore_electrolysers_tyndp( - n, pyear, offshore_electrolysers_fn, costs, logger, nyears - ) + add_offshore_electrolysers_tyndp(n, pyear, offshore_electrolysers_fn, costs, nyears) # Add offshore DC and H2 grid connections - add_offshore_grid_tyndp(n, pyear, offshore_grid_fn, costs, logger, nyears) + add_offshore_grid_tyndp(n, pyear, offshore_grid_fn, costs, nyears) def check_land_transport_shares(shares): @@ -7638,7 +7604,6 @@ def add_import_options( gas_input_nodes=gas_input_nodes, spatial=spatial, options=options, - logger=logger, ) if snakemake.params.offshore_hubs_tyndp: @@ -7652,7 +7617,6 @@ def add_import_options( pecd_mapping=pecd_mapping, costs=costs, spatial=spatial, - logger=logger, nyears=nyears, ) From 138600cbecafa2094731b8ceeae002fe0fb6c044 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 7 Jul 2025 11:52:56 +0200 Subject: [PATCH 141/165] feat: make the retrieve outputs more explicit --- rules/retrieve.smk | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 7a2f7824b6..2eae7b2595 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -181,6 +181,12 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" h2_imports="data/tyndp_2024_bundle/Hydrogen/H2 IMPORTS GENERATORS PROPERTIES.xlsx", offshore_nodes="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", offshore_grid="data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx", + offshore_electrolysers=directory( + "data/tyndp_2024_bundle/Offshore hubs/ELECTROLYSER.xlsx" + ), + offshore_generators=directory( + "data/tyndp_2024_bundle/Offshore hubs/GENERATOR.xlsx" + ), log: "logs/retrieve_tyndp_bundle.log", retries: 2 From 05c6523ce08bfc54d394d1c5ca0f8415cc3631a1 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 7 Jul 2025 13:44:27 +0200 Subject: [PATCH 142/165] fix: fix outputs in retrieve_tyndp_bundle --- rules/retrieve.smk | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 2eae7b2595..9f35a6b069 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -181,12 +181,8 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" h2_imports="data/tyndp_2024_bundle/Hydrogen/H2 IMPORTS GENERATORS PROPERTIES.xlsx", offshore_nodes="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", offshore_grid="data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx", - offshore_electrolysers=directory( - "data/tyndp_2024_bundle/Offshore hubs/ELECTROLYSER.xlsx" - ), - offshore_generators=directory( - "data/tyndp_2024_bundle/Offshore hubs/GENERATOR.xlsx" - ), + offshore_electrolysers="data/tyndp_2024_bundle/Offshore hubs/ELECTROLYSER.xlsx", + offshore_generators="data/tyndp_2024_bundle/Offshore hubs/GENERATOR.xlsx", log: "logs/retrieve_tyndp_bundle.log", retries: 2 From 9926a23984b659748bc27ce55dcf974f4bfe58d9 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 7 Jul 2025 14:04:35 +0200 Subject: [PATCH 143/165] refactor: use explicit list of tyndp carriers in constraint --- rules/solve_electricity.smk | 1 + rules/solve_myopic.smk | 1 + rules/solve_overnight.smk | 1 + rules/solve_perfect.smk | 1 + scripts/solve_network.py | 25 ++++++++++++++++++++----- 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 8c9b7f8867..8df3a2705e 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -11,6 +11,7 @@ rule solve_network: "sector", "co2_sequestration_potential", default=200 ), custom_extra_functionality=input_custom_extra_functionality, + carriers_tyndp=config_provider("electricity", "tyndp_renewable_carriers"), input: network=resources("networks/base_s_{clusters}_elec_{opts}.nc"), offshore_zone_trajectories=lambda w: ( diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index ea7daffa83..4e21907c6a 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -131,6 +131,7 @@ rule solve_sector_network_myopic: "sector", "co2_sequestration_potential", default=200 ), custom_extra_functionality=input_custom_extra_functionality, + carriers_tyndp=config_provider("electricity", "tyndp_renewable_carriers"), input: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc" diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 73364dcbb9..e24d93cbee 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -11,6 +11,7 @@ rule solve_sector_network: "sector", "co2_sequestration_potential", default=200 ), custom_extra_functionality=input_custom_extra_functionality, + carriers_tyndp=config_provider("electricity", "tyndp_renewable_carriers"), input: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 28b2aff166..2761b8b18f 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -101,6 +101,7 @@ rule solve_sector_network_perfect: "sector", "co2_sequestration_potential", default=200 ), custom_extra_functionality=input_custom_extra_functionality, + carriers_tyndp=config_provider("electricity", "tyndp_renewable_carriers"), input: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc" diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a2399e9856..2c1012c81c 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1135,7 +1135,10 @@ def add_import_limit_constraint(n: pypsa.Network, sns: pd.DatetimeIndex): def add_offshore_hubs_constraint( - n, planning_horizons: int | None, offshore_zone_trajectories_fn + n, + planning_horizons: int | None, + offshore_zone_trajectories_fn, + carriers_tyndp: list[str], ): """ Add two constraints on offshore hubs. @@ -1151,6 +1154,8 @@ def add_offshore_hubs_constraint( The current planning horizon year or None in perfect foresight offshore_zone_trajectories_fn: str Path to the file containing the offshore zone potentials trajectories + carriers_tyndp : list[str], optional + List of TYNDP renewable carriers """ ext_i = n.generators.p_nom_extendable gens = n.generators.assign( @@ -1159,9 +1164,11 @@ def add_offshore_hubs_constraint( ), zone=lambda df: df.index.str.split().str[0], ).rename_axis("Generator-ext") + off_carriers = [i for i in carriers_tyndp if "offwind" in i] + off_h2_carriers = [i for i in off_carriers if "h2" in i] # Constraint DC / H2 expansion on the same layer - h2_i = gens.carrier.str.contains("h2") + h2_i = gens.carrier.isin(off_h2_carriers) h2_gens = gens.loc[(h2_i) & (ext_i)] h2_gens_i = h2_gens.index dc_gens_i = h2_gens_i.str.replace("h2", "dc").str.replace(" H2", "") @@ -1186,7 +1193,7 @@ def add_offshore_hubs_constraint( .p_nom_max ) - off_i = gens.index.str.contains("offwind") + off_i = gens.carrier.isin(off_carriers) off_gens_i = gens.loc[(off_i) & (ext_i)].index grouper_ext = gens.loc[off_gens_i].zone.rename("Generator-ext") @@ -1241,6 +1248,7 @@ def extra_functionality( snapshots: pd.DatetimeIndex, planning_horizons: str | None = None, offshore_zone_trajectories_fn: str | None = None, + carriers_tyndp: list[str] = [], ) -> None: """ Add custom constraints and functionality. @@ -1255,6 +1263,8 @@ def extra_functionality( The current planning horizon year or None in perfect foresight offshore_zone_trajectories_fn: str, optional Path to the file containing the offshore zone potentials trajectories + carriers_tyndp : list[str], optional + List of TYNDP renewable carriers Collects supplementary constraints which will be passed to ``pypsa.optimization.optimize``. @@ -1313,7 +1323,7 @@ def extra_functionality( if config["sector"]["offshore_hubs_tyndp"]["enable"]: add_offshore_hubs_constraint( - n, int(planning_horizons), offshore_zone_trajectories_fn + n, int(planning_horizons), offshore_zone_trajectories_fn, carriers_tyndp ) if n.params.custom_extra_functionality: @@ -1362,6 +1372,7 @@ def solve_network( rule_name: str | None = None, planning_horizons: str | None = None, offshore_zone_trajectories_fn: str | None = None, + carriers_tyndp: list[str] = [], **kwargs, ) -> None: """ @@ -1380,9 +1391,11 @@ def solve_network( rule_name : str, optional Name of the snakemake rule being executed planning_horizons : str, optional - The current planning horizon year or None in perfect foresight + The current planning horizon year or None in perfect foresight offshore_zone_trajectories_fn : str, optional Path to DataFrame containing the offshore zone potentials trajectories + carriers_tyndp : list[str], optional + List of TYNDP renewable carriers **kwargs Additional keyword arguments passed to the solver @@ -1414,6 +1427,7 @@ def solve_network( extra_functionality, planning_horizons=planning_horizons, offshore_zone_trajectories_fn=offshore_zone_trajectories_fn, + carriers_tyndp=carriers_tyndp, ) kwargs["transmission_losses"] = cf_solving.get("transmission_losses", False) kwargs["linearized_unit_commitment"] = cf_solving.get( @@ -1520,6 +1534,7 @@ def solve_network( rule_name=snakemake.rule, log_fn=snakemake.log.solver, offshore_zone_trajectories_fn=snakemake.input.offshore_zone_trajectories, + carriers_tyndp=snakemake.params.carriers_tyndp, ) logger.info(f"Maximum memory usage: {mem.mem_usage}") From 53b0547d6ba2398ac868dec77219b9de0fbe004c Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 7 Jul 2025 15:03:41 +0200 Subject: [PATCH 144/165] refactor: apply code suggestion --- config/config.tyndp.yaml | 3 +++ config/test/config.tyndp.yaml | 3 +++ scripts/add_brownfield.py | 3 +-- scripts/prepare_sector_network.py | 14 ++++++-------- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index b354c4f7e2..753b4f5671 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -150,6 +150,9 @@ sector: - H2 offshore_hubs_tyndp: enable: true + max_capacity: + DC: 10 + H2: 30 costs: overwrites: diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 4abad2f1e2..3f9b823e20 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -176,6 +176,9 @@ sector: - H2 offshore_hubs_tyndp: enable: true + max_capacity: + DC: 10 + H2: 30 costs: overwrites: diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 6566417093..0e057170ab 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -138,6 +138,7 @@ def add_brownfield( ) # account for the shared potential of hydrogen- and electricity-generating wind farms + already_existing_l = already_existing.copy() if c.name == "Generator": h2_gens = already_existing.loc[ already_existing.index.str.contains("h2") @@ -176,8 +177,6 @@ def add_brownfield( .groupby(level=0) .sum() ) - else: - already_existing_l = already_existing.copy() # values should be non-negative; clipping applied to handle rounding errors remaining_capacity = ( diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index aaeb390bf2..ba07ed8555 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3067,14 +3067,12 @@ def add_offshore_generators_tyndp( # Adjust capacities and costs to account for efficiency h2_idx = offshore_generators.filter(like="h2", axis=0).index - offshore_generators.loc[h2_idx, ["p_nom_min", "p_nom_max"]] = ( - offshore_generators.loc[h2_idx, ["p_nom_min", "p_nom_max"]].mul( - costs.at["electrolysis", "efficiency"] - ) - ) - offshore_generators.loc[h2_idx, ["capex", "opex"]] = offshore_generators.loc[ - h2_idx, ["capex", "opex"] - ].div(costs.at["electrolysis", "efficiency"]) + offshore_generators.loc[h2_idx, ["p_nom_min", "p_nom_max"]] *= costs.at[ + "electrolysis", "efficiency" + ] + offshore_generators.loc[h2_idx, ["capex", "opex"]] /= costs.at[ + "electrolysis", "efficiency" + ] # Determine capital_cost annuity_factor = calculate_annuity(costs["lifetime"], costs["discount rate"]) From 982071f5e0e1edd02a73c6b7b63cd3e5ea3a5e12 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 7 Jul 2025 16:03:09 +0200 Subject: [PATCH 145/165] refactor: improve code --- scripts/build_tyndp_offshore_hubs.py | 19 +++++++++++++------ scripts/solve_network.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index b84beb28ba..501e7aa450 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -10,9 +10,10 @@ import geopandas as gpd import numpy as np import pandas as pd -from _helpers import configure_logging, set_scenario_config from shapely.geometry import Point +from scripts._helpers import configure_logging, set_scenario_config + logger = logging.getLogger(__name__) GEO_CRS = "EPSG:4326" @@ -151,7 +152,7 @@ def load_offshore_grid( p_max_pu=1, ) - # Assume non-extendable when missing data + # Handle missing cost data # TODO Validate assumption grid["p_nom_extendable"] = ~grid[["capex", "opex"]].isna().any(axis=1) grid[["capex", "opex"]] = grid[["capex", "opex"]].fillna(0) @@ -460,16 +461,22 @@ def load_generators(sheet_name, tech_switch=None): # Collect potentials trajectories in ZONE_POTENTIAL zone_trajectories = generators_z - # Resolve discrepancy in DEOH002 + # Resolve known DEOH002 data discrepancy + # This is a temporary fix for a 526 MW discrepancy between LAYER_POTENTIAL + # and ZONE_POTENTIAL data sources. + # TODO: Remove this once upstream TYNDP data is corrected + DEOH002_DISCREPANCY_MW = 526 idx_l = generators.query( "location=='DEOH002' and pyear == 2040 and carrier=='offwind-ac-fb-r'" ).index - generators.loc[idx_l, "p_nom_min"] = generators.loc[idx_l, "p_nom_min"] - 526 + generators.loc[idx_l, "p_nom_min"] = ( + generators.loc[idx_l, "p_nom_min"] - DEOH002_DISCREPANCY_MW + ) idx_z = zone_trajectories.query( "location=='DEOH002' and pyear in [2045, 2050]" ).index zone_trajectories.loc[idx_z, "p_nom_max"] = ( - zone_trajectories.loc[idx_z, "p_nom_max"] - 526 + zone_trajectories.loc[idx_z, "p_nom_max"] - DEOH002_DISCREPANCY_MW ) # Collect cost assumptions @@ -505,7 +512,7 @@ def load_generators(sheet_name, tech_switch=None): if __name__ == "__main__": if "snakemake" not in globals(): - from _helpers import mock_snakemake + from scripts._helpers import mock_snakemake snakemake = mock_snakemake( "build_tyndp_offshore_hubs", configfiles="config/test/config.tyndp.yaml" diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 2c1012c81c..59c9124ce2 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1136,7 +1136,7 @@ def add_import_limit_constraint(n: pypsa.Network, sns: pd.DatetimeIndex): def add_offshore_hubs_constraint( n, - planning_horizons: int | None, + planning_horizons: int, offshore_zone_trajectories_fn, carriers_tyndp: list[str], ): From 5306440f83373ad5ec6e79919d67119daa9e42da Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Mon, 7 Jul 2025 16:54:27 +0200 Subject: [PATCH 146/165] doc: adjust licensing --- rules/solve_electricity.smk | 2 +- rules/solve_overnight.smk | 2 +- scripts/build_tyndp_offshore_hubs.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 8df3a2705e..29816afc89 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH and contributors to PyPSA-Eur # # SPDX-License-Identifier: MIT diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index e24d93cbee..034fa73fbd 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH and contributors to PyPSA-Eur # # SPDX-License-Identifier: MIT diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 501e7aa450..913551ddba 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: : Open Energy Transition gGmbH +# SPDX-FileCopyrightText: Open Energy Transition gGmbH # # SPDX-License-Identifier: MIT """ From a6bf6e1b33a37171a57b9604b718693a3df749f2 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 9 Jul 2025 11:34:01 +0200 Subject: [PATCH 147/165] doc: make README more explicit about make tyndp --- README.md | 6 +++--- Snakefile | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d9623ae6bd..cea9e0720e 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,13 @@ Activate the newly created `open-tydnp` environment: ## 2. Run the analysis - snakemake -call + make tyndp This will run all analysis steps to reproduce results and build the report. -To generate a PDF of the dependency graph of all steps `resources/dag.pdf` run: +To generate a PDF of the dependency graph of all steps `resources/dag_rulegraph.pdf` run: - snakemake -c1 dag + snakemake -c1 rulegraph * Open Energy Transition (g)GmbH, Königsallee 52, 95448 Bayreuth, Germany diff --git a/Snakefile b/Snakefile index a4f3f7b500..1488749cb0 100644 --- a/Snakefile +++ b/Snakefile @@ -235,7 +235,7 @@ rule rulegraph: r""" # Generate DOT file using nested snakemake with the dumped final config echo "[Rule rulegraph] Using final config file: {input.config_file}" - snakemake --rulegraph all --configfile {input.config_file} --quiet | sed -n "/digraph/,\$p" > {output.dot} + snakemake --rulegraph --configfile {input.config_file} --quiet | sed -n "/digraph/,\$p" > {output.dot} # Generate visualizations from the DOT file if [ -s {output.dot} ]; then From 89b255ef8401943c1b6ac7f2b1430fa3679b393b Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 15 Jul 2025 17:35:07 +0200 Subject: [PATCH 148/165] refactor: define a specific tyndp all function --- Snakefile | 76 +++++++++++++++++++++++++------------------ rules/postprocess.smk | 2 +- 2 files changed, 45 insertions(+), 33 deletions(-) diff --git a/Snakefile b/Snakefile index 1488749cb0..995efc2781 100644 --- a/Snakefile +++ b/Snakefile @@ -77,22 +77,10 @@ if config["foresight"] == "perfect": include: "rules/solve_perfect.smk" -rule all: - input: - expand(RESULTS + "graphs/costs.svg", run=config["run"]["name"]), - expand(resources("maps/power-network.pdf"), run=config["run"]["name"]), - expand( - resources("maps/power-network-s-{clusters}.pdf"), - run=config["run"]["name"], - **config["scenario"], - ), +def input_all_tyndp(w): + files = [] + files.extend( expand( - RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", - run=config["run"]["name"], - **config["scenario"], - ), - lambda w: expand( ( resources( "maps/base_h2_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" @@ -102,49 +90,73 @@ rule all: ), run=config["run"]["name"], **config["scenario"], - ), - lambda w: expand( + ) + ) + files.extend( + expand( ( - RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-h2_network.pdf" - if config_provider("sector", "H2_network")(w) + resources( + "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.pdf" + ) + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) else [] ), run=config["run"]["name"], **config["scenario"], - ), - lambda w: expand( + carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), + ) + ) + files.extend( + expand( ( RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf" - if config_provider("sector", "gas_network")(w) + + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network_{carrier}.pdf" + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) else [] ), run=config["run"]["name"], **config["scenario"], + carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), + ) + ) + return files + + +rule all: + input: + input_all_tyndp, + expand(RESULTS + "graphs/costs.svg", run=config["run"]["name"]), + expand(resources("maps/power-network.pdf"), run=config["run"]["name"]), + expand( + resources("maps/power-network-s-{clusters}.pdf"), + run=config["run"]["name"], + **config["scenario"], + ), + expand( + RESULTS + + "maps/base_s_{clusters}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", + run=config["run"]["name"], + **config["scenario"], ), lambda w: expand( ( - resources( - "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.pdf" - ) - if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) + RESULTS + + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-h2_network.pdf" + if config_provider("sector", "H2_network")(w) else [] ), run=config["run"]["name"], **config["scenario"], - carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), ), lambda w: expand( ( RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network_{carrier}.pdf" - if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) + + "maps/base_s_{clusters}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf" + if config_provider("sector", "gas_network")(w) else [] ), run=config["run"]["name"], **config["scenario"], - carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), ), lambda w: expand( ( diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 30e7bfc202..9cd1193b0a 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -110,7 +110,7 @@ if config["foresight"] != "perfect": ), output: map=RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-h2_network.pdf", + + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf", threads: 2 resources: mem_mb=10000, From 199dbcaae3bd7461029c672bd42f0b5db8502a66 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 15 Jul 2025 17:44:06 +0200 Subject: [PATCH 149/165] refactor: prefer branch over lambda formulation --- rules/solve_electricity.smk | 7 +++---- rules/solve_myopic.smk | 7 +++---- rules/solve_overnight.smk | 7 +++---- rules/solve_perfect.smk | 7 +++---- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/rules/solve_electricity.smk b/rules/solve_electricity.smk index 29816afc89..fd89175f77 100644 --- a/rules/solve_electricity.smk +++ b/rules/solve_electricity.smk @@ -14,10 +14,9 @@ rule solve_network: carriers_tyndp=config_provider("electricity", "tyndp_renewable_carriers"), input: network=resources("networks/base_s_{clusters}_elec_{opts}.nc"), - offshore_zone_trajectories=lambda w: ( - resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) - else [] + offshore_zone_trajectories=branch( + config_provider("sector", "offshore_hubs_tyndp", "enable"), + resources("offshore_zone_trajectories.csv"), ), output: network=RESULTS + "networks/base_s_{clusters}_elec_{opts}.nc", diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 4e21907c6a..7c27cfdf0c 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -137,10 +137,9 @@ rule solve_sector_network_myopic: "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}_brownfield.nc" ), costs=resources("costs_{planning_horizons}.csv"), - offshore_zone_trajectories=lambda w: ( - resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) - else [] + offshore_zone_trajectories=branch( + config_provider("sector", "offshore_hubs_tyndp", "enable"), + resources("offshore_zone_trajectories.csv"), ), output: network=RESULTS diff --git a/rules/solve_overnight.smk b/rules/solve_overnight.smk index 034fa73fbd..c521c20774 100644 --- a/rules/solve_overnight.smk +++ b/rules/solve_overnight.smk @@ -16,10 +16,9 @@ rule solve_sector_network: network=resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" ), - offshore_zone_trajectories=lambda w: ( - resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) - else [] + offshore_zone_trajectories=branch( + config_provider("sector", "offshore_hubs_tyndp", "enable"), + resources("offshore_zone_trajectories.csv"), ), output: network=RESULTS diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 2761b8b18f..09001ce592 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -107,10 +107,9 @@ rule solve_sector_network_perfect: "networks/base_s_{clusters}_{opts}_{sector_opts}_brownfield_all_years.nc" ), costs=resources("costs_2030.csv"), - offshore_zone_trajectories=lambda w: ( - resources("offshore_zone_trajectories.csv") - if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) - else [] + offshore_zone_trajectories=branch( + config_provider("sector", "offshore_hubs_tyndp", "enable"), + resources("offshore_zone_trajectories.csv"), ), output: network=RESULTS From d12c611846dd7239b51f91979a120f71c796c751 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 15 Jul 2025 18:10:56 +0200 Subject: [PATCH 150/165] refactor: apply code suggestions --- rules/build_sector.smk | 10 ++++------ scripts/build_tyndp_offshore_hubs.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 5696dcdcd5..2d73873922 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1375,12 +1375,10 @@ if config["sector"]["offshore_hubs_tyndp"]["enable"]: countries=config_provider("countries"), offshore_hubs_tyndp=config_provider("sector", "offshore_hubs_tyndp"), input: - nodes=directory("data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx"), - grid=directory("data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx"), - electrolysers=directory( - "data/tyndp_2024_bundle/Offshore hubs/ELECTROLYSER.xlsx" - ), - generators=directory("data/tyndp_2024_bundle/Offshore hubs/GENERATOR.xlsx"), + nodes="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", + grid="data/tyndp_2024_bundle/Offshore hubs/GRID.xlsx", + electrolysers="data/tyndp_2024_bundle/Offshore hubs/ELECTROLYSER.xlsx", + generators="data/tyndp_2024_bundle/Offshore hubs/GENERATOR.xlsx", output: offshore_buses=resources("offshore_buses.csv"), offshore_grid=resources("offshore_grid.csv"), diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 913551ddba..fa3de20f95 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -10,7 +10,6 @@ import geopandas as gpd import numpy as np import pandas as pd -from shapely.geometry import Point from scripts._helpers import configure_logging, set_scenario_config @@ -53,8 +52,9 @@ def load_offshore_hubs(fn: str, countries: list[str]): ) ) - nodes["geometry"] = nodes.apply(lambda row: Point(row["x"], row["y"]), axis=1) - nodes = gpd.GeoDataFrame(nodes, geometry="geometry", crs=GEO_CRS) + nodes = gpd.GeoDataFrame( + nodes, geometry=gpd.points_from_xy(nodes.x, nodes.y), crs=GEO_CRS + ) return nodes @@ -120,11 +120,11 @@ def load_offshore_grid( } # Load reference grid - grid = pd.read_excel( - fn, - sheet_name="Reference grid", - ).rename(columns=column_dict) - grid["carrier"] = grid["carrier"].replace("E", "DC") + grid = ( + pd.read_excel(fn, sheet_name="Reference grid") + .rename(columns=column_dict) + .replace({"carrier": {"E": "DC"}, "scenario": scenario_dict}) + ) grid = expand_all_scenario(grid, scenario_dict.values()).query( "scenario == @scenario" ) From 07a78a47fac640445c2814e9b27219005b5ad5bd Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Wed, 16 Jul 2025 11:10:09 +0200 Subject: [PATCH 151/165] fix: fix file name in all rule --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index 995efc2781..d97fffe132 100644 --- a/Snakefile +++ b/Snakefile @@ -141,7 +141,7 @@ rule all: lambda w: expand( ( RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-h2_network.pdf" + + "maps/base_s_{clusters}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf" if config_provider("sector", "H2_network")(w) else [] ), From d4493fe8f8886ef91482fed7bfa2d328176a7d84 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 17 Jul 2025 13:34:06 +0200 Subject: [PATCH 152/165] refactor: remove unused function arguments --- scripts/build_tyndp_offshore_hubs.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index fa3de20f95..3a0f7228a8 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -18,7 +18,7 @@ GEO_CRS = "EPSG:4326" -def load_offshore_hubs(fn: str, countries: list[str]): +def load_offshore_hubs(fn: str): """ Load and process offshore hub coordinates from Excel file. @@ -26,8 +26,6 @@ def load_offshore_hubs(fn: str, countries: list[str]): ---------- fn : str Path to the Excel file containing offshore hub data. - countries : list[str] - List of country codes used to clean data. Returns ------- @@ -71,7 +69,6 @@ def expand_all_scenario(df: pd.DataFrame, scenarios: list): def load_offshore_grid( fn: str, - nodes: pd.DataFrame, scenario: str, planning_horizons: list[int], countries: list[str], @@ -84,8 +81,6 @@ def load_offshore_grid( ---------- fn : str Path to the Excel file containing offshore grid data. - nodes : pd.DataFrame - DataFrame containing node information. scenario : str Scenario identifier to filter the grid data. Must be one of the scenario codes: "DE" (Distributed Energy), "GA" (Global Ambition), or @@ -526,11 +521,10 @@ def load_generators(sheet_name, tech_switch=None): planning_horizons = snakemake.params["planning_horizons"] countries = snakemake.params["countries"] - nodes = load_offshore_hubs(snakemake.input.nodes, countries) + nodes = load_offshore_hubs(snakemake.input.nodes) grid = load_offshore_grid( snakemake.input.grid, - nodes, snakemake.params["scenario"], planning_horizons, countries, From aa7036ec43d94f96bdff40b36b2bed4a075f05b6 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 17 Jul 2025 13:52:51 +0200 Subject: [PATCH 153/165] refactor: update carrier naming to explicitly separate offshore hubs and DRES --- config/config.default.yaml | 4 ++-- config/config.tyndp.yaml | 4 ++-- config/plotting.default.yaml | 9 +++++++-- config/test/config.tyndp.yaml | 4 ++-- scripts/add_brownfield.py | 4 +--- scripts/build_tyndp_offshore_hubs.py | 13 ++++++++++--- scripts/plot_offshore_network.py | 28 ++++++++++++---------------- scripts/prepare_sector_network.py | 22 ++++++++++++---------- 8 files changed, 48 insertions(+), 40 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index ab88dfa657..c5c8e92ca6 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -820,8 +820,8 @@ sector: offshore_hubs_tyndp: enable: false max_capacity: - DC: 10 - H2: 30 + DC_OH: 10 + H2 pipeline OH: 30 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry industry: diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 753b4f5671..1b2fef2b3d 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -151,8 +151,8 @@ sector: offshore_hubs_tyndp: enable: true max_capacity: - DC: 10 - H2: 30 + DC_OH: 10 + H2 pipeline OH: 30 costs: overwrites: diff --git a/config/plotting.default.yaml b/config/plotting.default.yaml index f0b64a8844..98de8d3df8 100644 --- a/config/plotting.default.yaml +++ b/config/plotting.default.yaml @@ -243,8 +243,8 @@ plotting: branch_sizes: offshore_maps: bus_carriers: - - DC - - H2 + - DC_OH + - H2 pipeline OH nice_names: OCGT: "Open-Cycle Gas" @@ -494,6 +494,7 @@ plotting: H2 for industry: "#f073da" H2 for shipping: "#ebaee0" H2: '#bf13a0' + H2_OH: '#bf13a0' hydrogen: '#bf13a0' retrofitted H2 boiler: '#e5a0d9' SMR: '#870c71' @@ -505,6 +506,7 @@ plotting: H2 storage: '#bf13a0' land transport fuel cell: '#6b3161' H2 pipeline: '#f081dc' + H2 pipeline OH: '#f081dc' H2 pipeline retrofitted: '#ba99b5' H2 import LH2: "#F28C28" H2 import Pipeline: "#FFA500" @@ -577,11 +579,14 @@ plotting: geothermal district heat: '#d19D00' geothermal organic rankine cycle: '#ffbf00' AC: "#70af1d" + AC_OH: "#70af1d" + AC_DRES: "#70af1d" AC-AC: "#70af1d" AC line: "#70af1d" links: "#8a1caf" HVDC links: "#8a1caf" DC: "#8a1caf" + DC_OH: "#8a1caf" DC-DC: "#8a1caf" DC link: "#8a1caf" load: "#dd2e23" diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 3f9b823e20..5c526a71da 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -177,8 +177,8 @@ sector: offshore_hubs_tyndp: enable: true max_capacity: - DC: 10 - H2: 30 + DC_OH: 10 + H2 pipeline OH: 30 costs: overwrites: diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 0e057170ab..029b87da75 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -59,9 +59,7 @@ def add_brownfield( # electric transmission grid set optimised capacities of previous as minimum n.lines.s_nom_min = n_p.lines.s_nom_opt - dc_i = n.links[ - (n.links.carrier == "DC") & ~(n.links.index.str.contains("Offshore")) - ].index + dc_i = n.links[n.links.carrier == "DC"].index n.links.loc[dc_i, "p_nom_min"] = n_p.links.loc[dc_i, "p_nom_opt"] for c in n_p.iterate_components(["Link", "Generator", "Store"]): diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 3a0f7228a8..d3b511b683 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -118,7 +118,12 @@ def load_offshore_grid( grid = ( pd.read_excel(fn, sheet_name="Reference grid") .rename(columns=column_dict) - .replace({"carrier": {"E": "DC"}, "scenario": scenario_dict}) + .replace( + { + "carrier": {"E": "DC_OH", "H2": "H2 pipeline OH"}, + "scenario": scenario_dict, + } + ) ) grid = expand_all_scenario(grid, scenario_dict.values()).query( "scenario == @scenario" @@ -137,7 +142,9 @@ def load_offshore_grid( grid_costs[["capex", "opex"]] = grid_costs[["capex", "opex"]].mul( 1e3 ) # kEUR/MW to EUR/MW - grid_costs["carrier"] = grid_costs["carrier"].replace("E", "DC") + grid_costs["carrier"] = grid_costs["carrier"].replace( + {"E": "DC_OH", "H2": "H2 pipeline OH"} + ) # Merge information grid = grid.merge( @@ -155,7 +162,7 @@ def load_offshore_grid( # Add maximum transmission capacities grid["p_nom_max"] = np.where( - grid.carrier == "DC", max_capacity["DC"], max_capacity["H2"] + grid.carrier == "DC_OH", max_capacity["DC_OH"], max_capacity["H2 pipeline OH"] ) # Rename UK in GB diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index 53551cb046..9dcca3ab95 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -26,7 +26,7 @@ def plot_offshore_map( map_opts, map_fn, planning_horizons, - carrier="DC", + carrier="DC_OH", p_nom="p_nom", legend=True, hubs_only=False, @@ -62,19 +62,17 @@ def plot_offshore_map( """ n = network.copy() - linewidth_factor = 1e4 - - mask = {"DC": "Offshore DC", "H2": "Offshore H2 pipeline"} + linewidth_factor = 1e4 if carrier == "DC_OH" else 5e3 n.links.drop( - n.links.index[~(n.links.index.str.contains(mask[carrier]))], + n.links.index[n.links.carrier != carrier], inplace=True, ) # transmission capacities if isinstance(p_nom, str): links = ( - n.links[n.links.index.str.contains(mask[carrier])][p_nom] + n.links[n.links.carrier == carrier][p_nom] .rename(index=lambda x: re.sub(r"-\d{4}$", f"-{planning_horizons}", x)) .groupby(level=0) .sum() @@ -91,18 +89,16 @@ def plot_offshore_map( raise ValueError("Parameter 'p_nom' must be either str or float.") # keep relevant buses - bus_carriers = [carrier] + (["AC"] if carrier == "DC" else []) + bus_carriers = [carrier.replace("DC", "AC")] + ( + ["AC"] if carrier == "DC_OH" else ["H2", "H2_OH"] + ) n.buses.drop( - n.buses.index[ - (~n.buses.carrier.isin(bus_carriers)) - | (n.buses.index.str.contains("Z1")) - | (n.buses.index.str.contains("DRES")) - ], + n.buses.index[~n.buses.carrier.isin(bus_carriers)], inplace=True, ) n_oh = n.copy() n_oh.buses.drop( - n_oh.buses.index[~n_oh.buses.index.str.contains("OH")], inplace=True + n_oh.buses.index[~n_oh.buses.carrier.str.contains("OH")], inplace=True ) if hubs_only: @@ -114,7 +110,7 @@ def plot_offshore_map( fig, ax = plt.subplots(figsize=(7, 6), subplot_kw={"projection": proj}) color_h2 = "#f081dc" color_dc = "darkseagreen" - color = color_dc if carrier == "DC" else color_h2 + color = color_dc if carrier == "DC_OH" else color_h2 color_oh_nodes = "#ff29d9" color_hm_nodes = "darkgray" @@ -189,7 +185,7 @@ def plot_offshore_map( legend_kw=legend_kw, ) - label = "DC link" if carrier == "DC" else "H2 pipeline" + label = "DC link" if carrier == "DC_OH" else "H2 pipeline" legend_kw = dict( loc="upper left", @@ -216,7 +212,7 @@ def plot_offshore_map( clusters="all", sector_opts="", planning_horizons=2050, - carrier="DC", + carrier="DC_OH", ) configure_logging(snakemake) set_scenario_config(snakemake) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index ba07ed8555..0d25f11d9f 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -86,6 +86,10 @@ def define_spatial(nodes, options, offshore_buses_fn=None, buses_h2_file=None): offshore_buses_h2 = offshore_buses.set_index(offshore_buses.index + " H2") spatial.offshore_hubs.nodes = offshore_buses.index spatial.offshore_hubs.nodes_h2 = offshore_buses_h2.index + spatial.offshore_hubs.carrier = pd.Series("AC_OH", index=offshore_buses.index) + spatial.offshore_hubs.carrier_h2 = pd.Series( + "H2_OH", index=offshore_buses_h2.index + ) spatial.offshore_hubs.x = offshore_buses.x spatial.offshore_hubs.y = offshore_buses.y spatial.offshore_hubs.x_h2 = offshore_buses_h2.x @@ -1649,9 +1653,7 @@ def insert_electricity_distribution_grid( - Micro-CHP units """ - nodes = n.buses.query( - "carrier == 'AC' and not index.str.contains('DRES') and not index.str.contains('OH')" - ).index + nodes = n.buses.query("carrier == 'AC'").index n.add( "Bus", @@ -1985,7 +1987,7 @@ def add_h2_dres_tyndp(n, spatial, buses_h2_z2, costs): location=buses_h2_z2, country=spatial.h2_tyndp.df.loc[buses_h2_z2].country.values, v_nom=380.0, - carrier="AC", + carrier="AC_DRES", unit="MWh_el", substation_off=1.0, substation_lv=1.0, @@ -3241,7 +3243,7 @@ def add_offshore_grid_tyndp( annuity_factor = calculate_annuity(costs["lifetime"], costs["discount rate"]) # Add DC grid connections - offshore_grid_dc = offshore_grid.query("carrier=='DC'").copy() + offshore_grid_dc = offshore_grid.query("carrier=='DC_OH'").copy() offshore_grid_dc.index = offshore_grid_dc.apply( lambda x: f"{x.bus0}-{x.bus1}-Offshore DC", axis=1 ) @@ -3262,12 +3264,12 @@ def add_offshore_grid_tyndp( p_min_pu=offshore_grid_dc.p_min_pu, p_max_pu=offshore_grid_dc.p_max_pu, capital_cost=offshore_grid_dc.capital_cost, - carrier="DC", + carrier=offshore_grid_dc.carrier, lifetime=costs.at["HVDC submarine", "lifetime"], ) # Add H2 pipeline connections - offshore_grid_h2 = offshore_grid.query("carrier=='H2'").copy() + offshore_grid_h2 = offshore_grid.query("carrier=='H2_OH'").copy() offshore_grid_h2 = offshore_grid_h2.assign( bus0=lambda df: np.where( df.bus0.str.contains("OH"), df.bus0 + " H2", df.bus0.str[:2] + " H2 Z2" @@ -3297,7 +3299,7 @@ def add_offshore_grid_tyndp( p_min_pu=offshore_grid_h2.p_min_pu, p_max_pu=offshore_grid_h2.p_max_pu, capital_cost=offshore_grid_h2.capital_cost, - carrier="H2 pipeline", + carrier=offshore_grid_h2.carrier, lifetime=costs.at["H2 (g) submarine pipeline", "lifetime"], ) @@ -3366,7 +3368,7 @@ def add_offshore_hubs_tyndp( location=spatial.offshore_hubs.locations, country=spatial.offshore_hubs.country, type=spatial.offshore_hubs.type, - carrier="AC", + carrier="AC_OH", unit="MWh_el", v_nom=380, ) @@ -3379,7 +3381,7 @@ def add_offshore_hubs_tyndp( location=spatial.offshore_hubs.locations_h2, country=spatial.offshore_hubs.country_h2, type=spatial.offshore_hubs.type_h2, - carrier="H2", + carrier="H2_OH", unit="MWh_LHV", ) From e67bc5122f8d12854a7244de1f000019d30a4227 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Thu, 17 Jul 2025 14:55:29 +0200 Subject: [PATCH 154/165] fix: fix: restrict max_capacity condition to inter-hubs connections and correct unit --- scripts/build_tyndp_offshore_hubs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index d3b511b683..0594df655a 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -162,8 +162,13 @@ def load_offshore_grid( # Add maximum transmission capacities grid["p_nom_max"] = np.where( - grid.carrier == "DC_OH", max_capacity["DC_OH"], max_capacity["H2 pipeline OH"] - ) + grid.bus0.str.contains("OH") & grid.bus1.str.contains("OH"), + np.where( + (grid.carrier == "DC_OH"), max_capacity["DC_OH"], max_capacity["H2 pipeline OH"] + ) + * 1e3, + grid.get("p_nom_max", np.inf), + ) # MW > GW # Rename UK in GB grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB", regex=True) From e44df091593bc50f9f9f6b47d10f5748d7a72deb Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 11:11:53 +0200 Subject: [PATCH 155/165] fix: fix dimension issue in add_brownfield --- scripts/add_brownfield.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 029b87da75..36b21ff259 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -175,6 +175,8 @@ def add_brownfield( .groupby(level=0) .sum() ) + else: + already_existing_l = already_existing_l.p_nom_opt # values should be non-negative; clipping applied to handle rounding errors remaining_capacity = ( From f13b8188ddd4011e47c4a29342bc35c60807a700 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 12:16:54 +0200 Subject: [PATCH 156/165] refactor: convert to explicit hydrogen buses sooner --- scripts/build_tyndp_offshore_hubs.py | 30 ++++++++++++++++++++++---- scripts/prepare_sector_network.py | 32 ++-------------------------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 0594df655a..28156c4378 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -154,6 +154,24 @@ def load_offshore_grid( p_max_pu=1, ) + # Filter out radial nodes and Convert to explicit hydrogen buses + grid = grid.query("~bus0.str.contains('OR') and ~bus1.str.contains('OR')").assign( + bus0=lambda df: np.where( + df.carrier == "H2 pipeline OH", + np.where( + df.bus0.str.contains("OH"), df.bus0 + " H2", df.bus0.str[:2] + " H2 Z2" + ), + df.bus0, + ), + bus1=lambda df: np.where( + df.carrier == "H2 pipeline OH", + np.where( + df.bus1.str.contains("OH"), df.bus1 + " H2", df.bus1.str[:2] + " H2 Z2" + ), + df.bus1, + ), + ) + # Handle missing cost data # TODO Validate assumption grid["p_nom_extendable"] = ~grid[["capex", "opex"]].isna().any(axis=1) @@ -164,7 +182,9 @@ def load_offshore_grid( grid["p_nom_max"] = np.where( grid.bus0.str.contains("OH") & grid.bus1.str.contains("OH"), np.where( - (grid.carrier == "DC_OH"), max_capacity["DC_OH"], max_capacity["H2 pipeline OH"] + (grid.carrier == "DC_OH"), + max_capacity["DC_OH"], + max_capacity["H2 pipeline OH"], ) * 1e3, grid.get("p_nom_max", np.inf), @@ -177,9 +197,7 @@ def load_offshore_grid( grid = grid.assign( country0=lambda x: x.bus0.str[:2], country1=lambda x: x.bus1.str[:2], - ).query( - "country0 in @countries and country1 in @countries and ~bus0.str.contains('OR') and ~bus1.str.contains('OR')" - ) + ).query("country0 in @countries and country1 in @countries") return grid @@ -493,6 +511,10 @@ def load_generators(sheet_name, tech_switch=None): on=["bus", "location", "pyear", "scenario", "type", "carrier"], ) + # Convert to explicit hydrogen buses + mask = generators["carrier"].str.contains("h2") + generators.loc[mask, "bus"] = generators.loc[mask, "bus"] + " H2" + # Validate that all required cost assumptions are defined if generators[["capex", "opex"]].isna().any().any(): raise ValueError("Missing generator cost data in input dataset.") diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 0d25f11d9f..4f86cd3423 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3061,8 +3061,6 @@ def add_offshore_generators_tyndp( offshore_generators = pd.read_csv(offshore_generators_fn).query("pyear==@pyear") # Assign locations and index - mask = offshore_generators["carrier"].str.contains("h2") - offshore_generators.loc[mask, "bus"] = offshore_generators.loc[mask, "bus"] + " H2" offshore_generators.index = ( offshore_generators.location + " 0 " + offshore_generators.carrier ) @@ -3083,7 +3081,7 @@ def add_offshore_generators_tyndp( + offshore_generators["opex"] ) * nyears - # mapping from TYNDP offshore generators to PECD profiles + # Mapping from TYNDP offshore generators to PECD profiles offshore_generators["pecd_profile_name"] = offshore_generators["carrier"].map( pecd_mapping ) @@ -3183,23 +3181,6 @@ def add_offshore_electrolysers_tyndp( ) -def map_h2_buses(n, df): - """ - Map AC buses to H2 Z2 buses. - """ - h2_busmap = ( - n.buses.query( - "~Bus.str.contains('DRES') and carrier=='AC' and type==''" - ).location.str[:2] - + " H2 Z2" - ) - df_mapped = df.assign( - bus0=lambda x: x["bus0"].map(h2_busmap).fillna(x["bus0"]), - bus1=lambda x: x["bus1"].map(h2_busmap).fillna(x["bus1"]), - ) - return df_mapped - - def add_offshore_grid_tyndp( n: pypsa.Network, pyear: int, @@ -3269,15 +3250,7 @@ def add_offshore_grid_tyndp( ) # Add H2 pipeline connections - offshore_grid_h2 = offshore_grid.query("carrier=='H2_OH'").copy() - offshore_grid_h2 = offshore_grid_h2.assign( - bus0=lambda df: np.where( - df.bus0.str.contains("OH"), df.bus0 + " H2", df.bus0.str[:2] + " H2 Z2" - ), - bus1=lambda df: np.where( - df.bus1.str.contains("OH"), df.bus1 + " H2", df.bus1.str[:2] + " H2 Z2" - ), - ) + offshore_grid_h2 = offshore_grid.query("carrier=='H2 pipeline OH'").copy() offshore_grid_h2.index = offshore_grid_h2.apply( make_index, axis=1, prefix="Offshore H2 pipeline" ) @@ -3285,7 +3258,6 @@ def add_offshore_grid_tyndp( annuity_factor.get("H2 (g) submarine pipeline") * offshore_grid_h2["capex"] + offshore_grid_h2["opex"] ) * nyears - offshore_grid_h2 = map_h2_buses(n, offshore_grid_h2) n.add( "Link", From a603ea25b9d279c28d9230a66a65113a394479c4 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 15:44:28 +0200 Subject: [PATCH 157/165] feat: add plotting threshold for plot_offshore_network --- scripts/plot_offshore_network.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index 9dcca3ab95..6500038b63 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -62,7 +62,8 @@ def plot_offshore_map( """ n = network.copy() - linewidth_factor = 1e4 if carrier == "DC_OH" else 5e3 + lw_factor = 1e4 if carrier == "DC_OH" else 5e3 + link_lower_threshold = 1e2 # MW below which not drawn n.links.drop( n.links.index[n.links.carrier != carrier], @@ -78,7 +79,8 @@ def plot_offshore_map( .sum() ) # set link widths - link_widths = links / linewidth_factor + links[links < link_lower_threshold] = 0.0 + link_widths = links / lw_factor if link_widths.notnull().empty: logger.info(f"No offshore capacities for {carrier}, skipping plot.") return @@ -137,7 +139,7 @@ def plot_offshore_map( if legend: sizes = [30, 10] labels = [f"{s} GW" for s in sizes] - scale = 1e3 / linewidth_factor + scale = 1e3 / lw_factor sizes = [s * scale for s in sizes] legend_kw = dict( From 35a67894bad03c61ea0d02c762b6fc3a31429cee Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 15:45:03 +0200 Subject: [PATCH 158/165] fix: use mainland GB coordinates instead of Northern Ireland --- scripts/build_tyndp_network.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/build_tyndp_network.py b/scripts/build_tyndp_network.py index ea6db8bcfd..db13786131 100644 --- a/scripts/build_tyndp_network.py +++ b/scripts/build_tyndp_network.py @@ -149,7 +149,7 @@ def build_shapes(bz_fn, geo_crs: str = GEO_CRS, distance_crs: str = DISTANCE_CRS y=lambda df: df["node"].y, ) - # Correct DK, IT, GR and SE coordinates + # Correct DK, IT, GR, SE and GB coordinates country_shapes.loc["DK", ["node", "x", "y"]] = bidding_shapes.loc[ "DKE1", ["node", "x", "y"] ] @@ -162,6 +162,9 @@ def build_shapes(bz_fn, geo_crs: str = GEO_CRS, distance_crs: str = DISTANCE_CRS country_shapes.loc["SE", ["node", "x", "y"]] = bidding_shapes.loc[ "SE01", ["node", "x", "y"] ] + country_shapes.loc["GB", ["node", "x", "y"]] = bidding_shapes.loc[ + "GB00", ["node", "x", "y"] + ] return bidding_shapes, country_shapes @@ -190,6 +193,7 @@ def build_buses( """ buses = ( pd.read_excel(buses_fn) + .replace("UK", "GB", regex=True) .merge( bidding_shapes[["country", "node", "x", "y"]], how="outer", From 0d1183e6c4a4d6c1d502d26e575ace8a4fc0ece3 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 16:10:46 +0200 Subject: [PATCH 159/165] fix: remove Offshore Hubs from hydrogen plots --- scripts/plot_base_hydrogen_network.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/plot_base_hydrogen_network.py b/scripts/plot_base_hydrogen_network.py index 8e3bc2248b..d42101b1a6 100644 --- a/scripts/plot_base_hydrogen_network.py +++ b/scripts/plot_base_hydrogen_network.py @@ -75,6 +75,10 @@ def plot_h2_map_base( ], inplace=True, ) + n.links.drop( + n.links.index[n.links.carrier.str.contains("OH")], + inplace=True, + ) p_nom = "p_nom_opt" if expanded else "p_nom" # capacity of pipes and imports @@ -100,7 +104,12 @@ def plot_h2_map_base( link_widths_imports = link_widths_imports.reindex(n.links.index).fillna(0.0) # drop non H2 buses - n.buses.drop(n.buses.index[~n.buses.carrier.str.contains("H2")], inplace=True) + n.buses.drop( + n.buses.index[ + (~n.buses.carrier.str.contains("H2")) | (n.buses.carrier.str.contains("OH")) + ], + inplace=True, + ) # optionally add hydrogen storage capacities onto the map if regions_for_storage is not None: From 7dc553d0eba7afcf14abc8c7f64db2a3fd9d3f1e Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 17:49:53 +0200 Subject: [PATCH 160/165] feat: introduce time-varying efficiencies for offwind generators (#68) * feat: introduce time-varying efficiencies for offwind generators * feat: use fixed 2020 efficiency values to prevent solver infeasibility * refactor: rename efficiency to efficiency_dc_to_b0 for offshore generators --- config/config.tyndp.yaml | 2 ++ config/test/config.tyndp.yaml | 2 ++ scripts/add_brownfield.py | 51 +++++++++++++++---------------- scripts/prepare_sector_network.py | 8 ++++- scripts/solve_network.py | 35 ++++++++++----------- 5 files changed, 52 insertions(+), 46 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 1b2fef2b3d..0dc36952a9 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -161,6 +161,8 @@ costs: HVDC submarine: 25 H2 (g) submarine pipeline: 25 offwind: 25 + efficiency: + electrolysis: 0.68 # ToDo Account for time-varying efficiencies, currently using fixed 2020 value from TYNDP 2024 clustering: mode: administrative diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 5c526a71da..0d664cd2cd 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -187,6 +187,8 @@ costs: HVDC submarine: 25 H2 (g) submarine pipeline: 25 offwind: 25 + efficiency: + electrolysis: 0.68 # ToDo Account for time-varying efficiencies, currently using fixed 2020 value from TYNDP 2024 clustering: mode: administrative diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 36b21ff259..6173f5a0c5 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -118,6 +118,7 @@ def add_brownfield( # hydrogen- and electricity-generating wind farms share the same potential; values are adjusted accordingly if offshore_hubs_tyndp: filter = {"Link": "Offshore", "Generator": "offwind"} + eff_map = {"Link": "efficiency", "Generator": "efficiency_dc_to_h2"} for c in n.iterate_components(["Link", "Generator"]): off_fixed_i = c.df[ (c.df.index.str.contains(filter[c.name])) & (c.df.build_year != year) @@ -128,9 +129,25 @@ def add_brownfield( off_capacity = c.df.loc[off_i, "p_nom"] off_potential = c.df.loc[off_i, "p_nom_max"] + + # Determine existing capacities in MW_e and MW_h2 already_existing = ( - c.df.loc[off_fixed_i, "p_nom_opt"] - .rename(lambda x: x.split("-2")[0] + f"-{year}") + c.df.loc[off_fixed_i] + .assign( + p_nom_opt_e=lambda df: np.where( + df.carrier.str.contains("h2"), + df.p_nom_opt.div(df[eff_map[c.name]]), + df.p_nom_opt, + ), + p_nom_opt_h2=lambda df: np.where( + ~df.carrier.str.contains("h2"), + df.p_nom_opt.mul(df[eff_map[c.name]]), + df.p_nom_opt, + ), + ) + .rename(lambda x: x.split("-2")[0] + f"-{year}")[ + ["p_nom_opt", "p_nom_opt_e", "p_nom_opt_h2"] + ] .groupby(level=0) .sum() ) @@ -145,33 +162,15 @@ def add_brownfield( already_existing.index.str.contains("dc.*oh") ] - off_h2_gens = n.generators.loc[h2_gens.index] - off_dc_gens = n.generators.loc[dc_gens.index] - off_electrolysers = n.links.loc[ - (n.links.index.str.contains("Offshore Electrolysis")) - & (n.links.build_year == year) - ].set_index("bus1") - # ToDo Account for time-varying efficiencies across planning horizons - eff_h2 = ( - off_electrolysers.loc[off_h2_gens.bus] - .set_index(h2_gens.index) - .efficiency - ) - eff_dc = ( - off_electrolysers.loc[off_dc_gens.bus + " H2"] - .set_index(dc_gens.index) - .efficiency - ) - - h2_to_dc = h2_gens.div(eff_h2).rename( + h2_to_dc = h2_gens.p_nom_opt_e.rename( index=lambda x: x.replace("h2", "dc") - ) - dc_to_h2 = dc_gens.mul(eff_dc).rename( + ).rename("p_nom_opt") + dc_to_h2 = dc_gens.p_nom_opt_h2.rename( index=lambda x: x.replace("dc", "h2") - ) + ).rename("p_nom_opt") already_existing_l = ( - pd.concat([already_existing, h2_to_dc, dc_to_h2]) + pd.concat([already_existing.p_nom_opt, h2_to_dc, dc_to_h2]) .groupby(level=0) .sum() ) @@ -181,7 +180,7 @@ def add_brownfield( # values should be non-negative; clipping applied to handle rounding errors remaining_capacity = ( off_capacity - - already_existing.reindex(index=off_capacity.index).fillna(0) + - already_existing.p_nom_opt.reindex(index=off_capacity.index).fillna(0) ).clip(lower=0) remaining_potential = ( off_potential diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 4f86cd3423..5b63ae6cad 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3067,6 +3067,11 @@ def add_offshore_generators_tyndp( # Adjust capacities and costs to account for efficiency h2_idx = offshore_generators.filter(like="h2", axis=0).index + offshore_generators["efficiency"] = np.where( + offshore_generators["carrier"].str.contains("h2"), + costs.at["electrolysis", "efficiency"], + 1.0, + ) offshore_generators.loc[h2_idx, ["p_nom_min", "p_nom_max"]] *= costs.at[ "electrolysis", "efficiency" ] @@ -3117,7 +3122,8 @@ def add_offshore_generators_tyndp( p_nom_extendable=offshore_generators.p_nom_extendable, capital_cost=offshore_generators.capital_cost, marginal_cost=costs.at["offwind", "marginal_cost"], - efficiency=costs.at["offwind", "efficiency"], + efficiency_dc_to_b0=offshore_generators.efficiency, + efficiency_dc_to_h2=costs.at["electrolysis", "efficiency"], p_max_pu=p_max_pu, lifetime=costs.at["offwind", "lifetime"], ) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 59c9124ce2..f6bb098061 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -1172,15 +1172,12 @@ def add_offshore_hubs_constraint( h2_gens = gens.loc[(h2_i) & (ext_i)] h2_gens_i = h2_gens.index dc_gens_i = h2_gens_i.str.replace("h2", "dc").str.replace(" H2", "") - - off_electrolysers = n.links.loc[ - (n.links.index.str.contains("Offshore Electrolysis")) - & (n.links.build_year == planning_horizons) - ].set_index("bus1") - eff_l = off_electrolysers.loc[h2_gens.bus].set_index(h2_gens_i).efficiency p_nom = n.model["Generator-p_nom"] - lhs = p_nom.loc[dc_gens_i] + p_nom.loc[h2_gens_i] / eff_l + lhs = ( + p_nom.loc[dc_gens_i] + + p_nom.loc[h2_gens_i] / h2_gens.loc[h2_gens_i, "efficiency_dc_to_b0"] + ) rhs = gens.loc[dc_gens_i].p_nom_max if not lhs.empty: @@ -1198,22 +1195,22 @@ def add_offshore_hubs_constraint( off_gens_i = gens.loc[(off_i) & (ext_i)].index grouper_ext = gens.loc[off_gens_i].zone.rename("Generator-ext") idx = pd.Index(set(limit.index).intersection(grouper_ext)) - eff_z = eff_l.reindex(off_gens_i, fill_value=1) + eff_z = gens["efficiency_dc_to_b0"].reindex(off_gens_i) lhs = (p_nom.loc[off_gens_i] / eff_z).groupby(grouper_ext).sum().loc[idx] - # ToDo Account for time-varying efficiencies across planning horizons - existing_z = ( - gens.loc[(off_i) & ~(ext_i), "p_nom"] - .rename(lambda x: x.split("-2")[0] + f"-{planning_horizons}") - .groupby(level=0) - .sum() - ) - grouper_z = gens.loc[existing_z.index].zone existing_z = ( - (existing_z / eff_z.loc[existing_z.index]) - .groupby(grouper_z) + gens.loc[(off_i) & ~(ext_i)] + .assign( + p_nom=lambda df: np.where( + df.carrier.str.contains("h2"), + df.p_nom.div(df.efficiency_dc_to_h2), + df.p_nom, + ) + ) + .rename(lambda x: x.split("-2")[0] + f"-{planning_horizons}")[["p_nom", "zone"]] + .groupby(by="zone") .sum() - .reindex(idx, fill_value=0) + .reindex(idx, fill_value=0)["p_nom"] ) rhs = limit.loc[idx] - existing_z From 4e29fb87f53d7a3fcc394b24294c17063f9c8832 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 16:57:34 +0200 Subject: [PATCH 161/165] refactor: move if outside of extend in Snakefile --- Snakefile | 70 ++++++++++++++++++++++++++----------------------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/Snakefile b/Snakefile index d97fffe132..cf5fb74ce4 100644 --- a/Snakefile +++ b/Snakefile @@ -79,46 +79,42 @@ if config["foresight"] == "perfect": def input_all_tyndp(w): files = [] - files.extend( - expand( - ( - resources( - "maps/base_h2_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" - ) - if config_provider("sector", "H2_network")(w) - else [] - ), - run=config["run"]["name"], - **config["scenario"], + if config_provider("sector", "H2_network")(w): + files.extend( + expand( + ( + resources( + "maps/base_h2_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}.pdf" + ) + ), + run=config["run"]["name"], + **config["scenario"], + ) ) - ) - files.extend( - expand( - ( - resources( - "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.pdf" - ) - if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) - else [] - ), - run=config["run"]["name"], - **config["scenario"], - carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w): + files.extend( + expand( + ( + resources( + "maps/base_offshore_network_{clusters}_{opts}_{sector_opts}_{planning_horizons}_{carrier}.pdf" + ) + ), + run=config["run"]["name"], + **config["scenario"], + carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), + ) ) - ) - files.extend( - expand( - ( - RESULTS - + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network_{carrier}.pdf" - if config_provider("sector", "offshore_hubs_tyndp", "enable")(w) - else [] - ), - run=config["run"]["name"], - **config["scenario"], - carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), + files.extend( + expand( + ( + RESULTS + + "maps/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}-offshore_network_{carrier}.pdf" + ), + run=config["run"]["name"], + **config["scenario"], + carrier=config_provider("plotting", "offshore_maps", "bus_carriers")(w), + ) ) - ) return files From 99ead1bb9066757024f6a83457ea54ccf2070559 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 17:05:29 +0200 Subject: [PATCH 162/165] refactor: remove redundant formulation in add_brownfield --- scripts/add_brownfield.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 6173f5a0c5..46535cec78 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -153,7 +153,6 @@ def add_brownfield( ) # account for the shared potential of hydrogen- and electricity-generating wind farms - already_existing_l = already_existing.copy() if c.name == "Generator": h2_gens = already_existing.loc[ already_existing.index.str.contains("h2") @@ -175,7 +174,7 @@ def add_brownfield( .sum() ) else: - already_existing_l = already_existing_l.p_nom_opt + already_existing_l = already_existing.p_nom_opt # values should be non-negative; clipping applied to handle rounding errors remaining_capacity = ( From f489dfa4f8dcfd8daf26e0a33f62cc9ac592d850 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 17:11:14 +0200 Subject: [PATCH 163/165] refactor: move input_offshore_hubs in file --- rules/build_sector.smk | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 2d73873922..eaaed26c7c 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1291,18 +1291,6 @@ def input_heat_source_power(w): } -def input_offshore_hubs(w): - offshore_files = [ - "offshore_buses", - "offshore_grid", - "offshore_electrolysers", - "offshore_generators", - ] - if config_provider("sector", "offshore_hubs_tyndp", "enable")(w): - return {f: resources(f"{f}.csv") for f in offshore_files} - return {f: [] for f in offshore_files} - - if config["sector"]["h2_topology_tyndp"]: rule build_tyndp_h2_network: @@ -1398,6 +1386,18 @@ if config["sector"]["offshore_hubs_tyndp"]["enable"]: "../scripts/build_tyndp_offshore_hubs.py" +def input_offshore_hubs(w): + offshore_files = [ + "offshore_buses", + "offshore_grid", + "offshore_electrolysers", + "offshore_generators", + ] + if config_provider("sector", "offshore_hubs_tyndp", "enable")(w): + return {f: resources(f"{f}.csv") for f in offshore_files} + return {f: [] for f in offshore_files} + + rule prepare_sector_network: params: time_resolution=config_provider("clustering", "temporal", "resolution_sector"), From 0ae5268a6893a7fb2256e371e2e350eccf2b2bc0 Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Fri, 18 Jul 2025 17:21:43 +0200 Subject: [PATCH 164/165] refactor: test before using input_offshore_hubs --- rules/build_sector.smk | 2 +- scripts/prepare_sector_network.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index eaaed26c7c..d2851ee557 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1395,7 +1395,7 @@ def input_offshore_hubs(w): ] if config_provider("sector", "offshore_hubs_tyndp", "enable")(w): return {f: resources(f"{f}.csv") for f in offshore_files} - return {f: [] for f in offshore_files} + return {} rule prepare_sector_network: diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 5b63ae6cad..d57b40505f 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -80,7 +80,7 @@ def define_spatial(nodes, options, offshore_buses_fn=None, buses_h2_file=None): # offshore hubs - if options.get("offshore_hubs_tyndp") and offshore_buses_fn: + if options["offshore_hubs_tyndp"]["enable"] and offshore_buses_fn: spatial.offshore_hubs = SimpleNamespace() offshore_buses = pd.read_csv(offshore_buses_fn, index_col=0) offshore_buses_h2 = offshore_buses.set_index(offshore_buses.index + " H2") @@ -7527,10 +7527,15 @@ def add_import_options( heating_efficiencies = pd.read_csv(fn, index_col=[1, 0]).loc[year] buses_h2_file = snakemake.input.buses_h2 if options["h2_topology_tyndp"] else None + buses_oh_file = ( + snakemake.input.offshore_buses + if options["offshore_hubs_tyndp"]["enable"] + else None + ) spatial = define_spatial( pop_layout.index, options, - offshore_buses_fn=snakemake.input.offshore_buses, + offshore_buses_fn=buses_oh_file, buses_h2_file=buses_h2_file, ) From a6eb8751837343d4462ea9ac99e073caf8bc395f Mon Sep 17 00:00:00 2001 From: Thomas Gilon Date: Tue, 22 Jul 2025 18:11:40 +0200 Subject: [PATCH 165/165] refactor: improve minor code inconsistencies --- scripts/build_tyndp_offshore_hubs.py | 51 +++++++++++----------------- scripts/plot_offshore_network.py | 2 +- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/scripts/build_tyndp_offshore_hubs.py b/scripts/build_tyndp_offshore_hubs.py index 28156c4378..9fdf4a0e32 100644 --- a/scripts/build_tyndp_offshore_hubs.py +++ b/scripts/build_tyndp_offshore_hubs.py @@ -16,6 +16,11 @@ logger = logging.getLogger(__name__) GEO_CRS = "EPSG:4326" +SCENARIO_DICT = { + "Distributed Energy": "DE", + "Global Ambition": "GA", + "National Trends": "NT", +} def load_offshore_hubs(fn: str): @@ -34,7 +39,7 @@ def load_offshore_hubs(fn: str): The GeoDataFrame uses the coordinate reference system defined by `GEO_CRS`. """ - column_dict = { + column_names = { "OFFSHORE_NODE": "Bus", "OFFSHORE_NODE_TYPE": "type", "LAT": "y", @@ -43,7 +48,7 @@ def load_offshore_hubs(fn: str): nodes = ( pd.read_excel(fn, sheet_name="NODE") - .rename(columns=column_dict) + .rename(columns=column_names) .assign( location=lambda x: x.Bus, country=lambda x: x.location.str[:2], @@ -97,7 +102,7 @@ def load_offshore_grid( pd.DataFrame DataFrame containing the merged offshore grid data. """ - column_dict = { + column_names = { "FROM": "bus0", "TO": "bus1", "YEAR": "pyear", @@ -108,24 +113,18 @@ def load_offshore_grid( "OPEX": "opex", } - scenario_dict = { - "Distributed Energy": "DE", - "Global Ambition": "GA", - "National Trends": "NT", - } - # Load reference grid grid = ( pd.read_excel(fn, sheet_name="Reference grid") - .rename(columns=column_dict) + .rename(columns=column_names) .replace( { "carrier": {"E": "DC_OH", "H2": "H2 pipeline OH"}, - "scenario": scenario_dict, + "scenario": SCENARIO_DICT, } ) ) - grid = expand_all_scenario(grid, scenario_dict.values()).query( + grid = expand_all_scenario(grid, SCENARIO_DICT.values()).query( "scenario == @scenario" ) @@ -135,8 +134,8 @@ def load_offshore_grid( fn, sheet_name="COST", ) - .rename(columns=column_dict) - .replace({"scenario": scenario_dict}) + .rename(columns=column_names) + .replace({"scenario": SCENARIO_DICT}) .query("pyear in @planning_horizons and scenario == @scenario") ) grid_costs[["capex", "opex"]] = grid_costs[["capex", "opex"]].mul( @@ -172,7 +171,7 @@ def load_offshore_grid( ), ) - # Handle missing cost data + # Handle missing data # TODO Validate assumption grid["p_nom_extendable"] = ~grid[["capex", "opex"]].isna().any(axis=1) grid[["capex", "opex"]] = grid[["capex", "opex"]].fillna(0) @@ -188,7 +187,7 @@ def load_offshore_grid( ) * 1e3, grid.get("p_nom_max", np.inf), - ) # MW > GW + ) # GW > MW # Rename UK in GB grid[["bus0", "bus1"]] = grid[["bus0", "bus1"]].replace("UK", "GB", regex=True) @@ -226,7 +225,7 @@ def load_offshore_electrolysers( pd.DataFrame DataFrame containing the formatted offshore electrolyser data. """ - column_dict = { + column_names = { "NODE": "bus0", "OFFSHORE_NODE_TYPE": "type", "YEAR": "pyear", @@ -235,21 +234,15 @@ def load_offshore_electrolysers( "OPEX": "opex", } - scenario_dict = { - "Distributed Energy": "DE", - "Global Ambition": "GA", - "National Trends": "NT", - } - # Load electrolysers data electrolysers = ( pd.read_excel( fn, sheet_name="COST", ) - .rename(columns=column_dict) + .rename(columns=column_names) .query("pyear in @planning_horizons") - .replace({"scenario": scenario_dict}) + .replace({"scenario": SCENARIO_DICT}) .query("scenario == @scenario") .assign(country=lambda x: x.bus0.str[:2], bus1=lambda x: x.bus0 + " H2") .drop(columns="OFFSHORE_NODE") @@ -438,12 +431,6 @@ def load_offshore_generators( "LAYER", ] - scenario_dict = { - "Distributed Energy": "DE", - "Global Ambition": "GA", - "National Trends": "NT", - } - # Load data def load_generators(sheet_name, tech_switch=None): generators = pd.read_excel( @@ -456,7 +443,7 @@ def load_generators(sheet_name, tech_switch=None): ) generators = ( generators.rename(columns=column_names) - .replace({"scenario": scenario_dict}) + .replace({"scenario": SCENARIO_DICT}) .query("pyear in @planning_horizons and scenario == @scenario") .assign( carrier=lambda x: "offwind-" diff --git a/scripts/plot_offshore_network.py b/scripts/plot_offshore_network.py index 6500038b63..3743f94cea 100644 --- a/scripts/plot_offshore_network.py +++ b/scripts/plot_offshore_network.py @@ -32,7 +32,7 @@ def plot_offshore_map( hubs_only=False, ): """ - Plots the offshore network hydrogen and electricity capacities and offshore-hubs buses. + Plots the offshore network hydrogen or electricity capacities and offshore-hubs buses. If `p_nom` parameter is set as `p_nom_opt`, optimal capacities are plotted instead. Parameters