From d2576ed47020def5d81543e3c99df196fe2d6002 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 21 Jul 2025 18:05:08 +0200 Subject: [PATCH 01/25] feat: introduce pipeline for hydro inflow data --- config/config.tyndp.yaml | 4 +- rules/build_electricity.smk | 52 +++++++++ rules/build_sector.smk | 6 + scripts/build_tyndp_hydro_profile.py | 18 +++ scripts/clean_tyndp_hydro_inflows.py | 165 +++++++++++++++++++++++++++ 5 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 scripts/build_tyndp_hydro_profile.py create mode 100644 scripts/clean_tyndp_hydro_inflows.py diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 45c07636fd..ef57279f25 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -40,8 +40,8 @@ co2_budget: electricity: base_network: tyndp-raw - renewable_carriers: [solar, solar-hsat, onwind, hydro] - tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + renewable_carriers: [solar, solar-hsat, onwind] + tyndp_renewable_carriers: [hydro, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 0797d3aef8..a6a17822ed 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -497,6 +497,58 @@ rule build_hydro_profile: "../scripts/build_hydro_profile.py" +rule clean_tyndp_hydro_inflows: + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + input: + hydro_inflows_dir="data/tyndp_2024_bundle/Hydro Inflows", + onshore_buses=resources("busmap_base_s_all.csv"), + output: + hydro_inflows_tyndp=resources( + "hydro_inflows_tyndp_{tech}_{planning_horizons}.csv" + ), + log: + logs("clean_tyndp_hydro_inflows_{tech}_{planning_horizons}.log"), + threads: 4 + benchmark: + benchmarks("clean_tyndp_hydro_inflows_{tech}_{planning_horizons}") + conda: + "../envs/environment.yaml" + script: + "../scripts/clean_tyndp_hydro_inflows.py" + + +def input_data_hydro_tyndp(w): + return { + f"hydro_inflow_tyndp_{tech}_{pyear}": resources( + f"hydro_inflows_tyndp_{tech}_{str(pyear)}.csv" + ) + for pyear in set( + config_provider("scenario", "planning_horizons")(w) + ).intersection([2030, 2040, 2050]) + # Hydro inflows data is only available for the years 2030, 2040, 2050 + for tech in ["Run of River", "Pondage", "Reservoir", "PS Open", "PS Closed"] + } + + +rule build_tyndp_hydro_profile: + input: + unpack(input_data_hydro_tyndp), + output: + profile=resources("profile_hydro_tyndp.nc"), + log: + logs("build_tyndp_hydro_profile.log"), + benchmark: + benchmarks("build_tyndp_hydro_profile") + resources: + mem_mb=5000, + conda: + "../envs/environment.yaml" + script: + "../scripts/build_tyndp_hydro_profile.py" + + rule build_line_rating: params: snapshots=config_provider("snapshots"), diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1a0a49872d..74e698278d 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1539,6 +1539,12 @@ rule prepare_sector_network: if config_provider("sector", "h2_topology_tyndp")(w) else [] ), + profile_hydro_tyndp=branch( + lambda w: "hydro" + in config_provider("electricity", "tyndp_renewable_carriers")(w), + resources("profile_hydro_tyndp.nc"), + [], + ), output: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" diff --git a/scripts/build_tyndp_hydro_profile.py b/scripts/build_tyndp_hydro_profile.py new file mode 100644 index 0000000000..8151e12bd3 --- /dev/null +++ b/scripts/build_tyndp_hydro_profile.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: MIT +""" +Build hydroelectric inflow time-series for each country based on TYNDP hydro inflow data. + +Outputs +------- + +- ``resources/profile_hydro_tyndp.nc``: + + =================== ================ ========================================================= + Field Dimensions Description + =================== ================ ========================================================= + inflow countries, time, Inflow to the state of charge (in MW), + year, hydro_tech e.g. due to river inflow in hydro reservoir. + =================== ================ ========================================================= +""" diff --git a/scripts/clean_tyndp_hydro_inflows.py b/scripts/clean_tyndp_hydro_inflows.py new file mode 100644 index 0000000000..c90e8f564e --- /dev/null +++ b/scripts/clean_tyndp_hydro_inflows.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: MIT +""" +Loads and cleans the available hydro inflow data from TYNDP data bundle for a given + +* climate year, +* planning horizon, +* hydro technology. + +Input data for TYNDP 2024 comes from PEMMDB v2.4. + +Outputs +------- +Cleaned csv file with hourly hydro inflow time series in MW per region. +""" + +import logging +import multiprocessing as mp +import os +from functools import partial +from pathlib import Path + +import numpy as np +import pandas as pd +from tqdm import tqdm + +from scripts._helpers import ( + configure_logging, + get_snapshots, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + + +def read_hydro_inflows_file( + node: str, + hydro_inflows_dir: str, + cyear: str, + pyear: str, + hydro_tech: str, + sns: pd.DatetimeIndex, +): + fn = Path(hydro_inflows_dir, pyear, f"PEMMDB_{node}_Hydro_Inflows_{pyear}.xlsx") + + if not os.path.isfile(fn): + return None + + tech_res = { + "Run of River": "d", + "Pondage": "d", + "Reservoir": "w", + "PS Open": "w", + "PS Closed": "w", + } + + inflow_tech = pd.read_excel( + fn, + skiprows=1, + usecols=lambda name: name == "Day" + or name == "Week" + or name == "ShortName" + or name == "Variable" + or name == int(cyear), + sheet_name=f"{hydro_tech} - Year Dependent", + ) + + sns_year = sns[0].year + date_index = { + "w": pd.date_range( + start=f"{sns_year}-01-01", + periods=53, # 53 weeks + freq="7D", + ), + "d": pd.date_range( + start=f"{sns_year}-01-01", + periods=366, # 366 days (incl. first day of next year) + freq="D", + ), + } + + inflow_tech = ( + inflow_tech.query("ShortName == 'INFLOW'") + .assign( + datetime=date_index[tech_res[hydro_tech]], + p_nom=lambda df: np.where( # calculate hourly inflow in GWh/h + df.Variable.str.contains("week"), + df[int(cyear)].div(24 * 7), # the value will be either in GWh/week + df[int(cyear)].div(24), # the value will be either in GWh/day + ), + ) + .set_index("datetime") + .reindex(sns) # filter for snapshots only + .ffill() + .div(1e3) # convert from GW to MW + .p_nom.rename(node) + ) + + return inflow_tech + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake( + "clean_tyndp_hydro_inflows", + clusters="all", + planning_horizons=2030, + tech="Run of River", + ) + configure_logging(snakemake) + set_scenario_config(snakemake) + + # Climate year from snapshots + sns = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + cyear = sns[0].year + if int(cyear) < 1982 or int(cyear) > 2019: + logger.warning( + "Snapshot year doesn't match available TYNDP data. Falling back to 2009." + ) + cyear = 2009 + + # Planning year + pyear = str(snakemake.wildcards.planning_horizons) + hydro_tech = str(snakemake.wildcards.tech) + + onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) + + nodes = onshore_buses.index.str.replace("GB", "UK", regex=True) + hydro_inflows_dir = snakemake.input.hydro_inflows_dir + + # Load and prep inflow data + tqdm_kwargs = { + "ascii": False, + "unit": " nodes", + "total": len(nodes), + "desc": "Loading TYNDP hydro inflows data", + } + + func = partial( + read_hydro_inflows_file, + hydro_inflows_dir=hydro_inflows_dir, + cyear=cyear, + pyear=pyear, + hydro_tech=hydro_tech, + sns=sns, + ) + + with mp.Pool(processes=snakemake.threads) as pool: + inflows = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + + inflows_df = ( + pd.concat(inflows, axis=1) + .reindex( + nodes, axis=1, fill_value=0.0 + ) # include missing node data with empty columns + .rename( + columns=lambda x: x.replace("UK", "GB") + ) # replace UK with GB for naming convention + .fillna(0.0) + ) + + inflows_df.to_csv(snakemake.output.hydro_inflows_tyndp) From 5edeafbda3a8bda29c526b3e8186ee0faf5cf627 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 24 Jul 2025 15:37:43 +0200 Subject: [PATCH 02/25] feat: infer resolution of input data for each technology and fix unit conversion --- scripts/clean_tyndp_hydro_inflows.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/scripts/clean_tyndp_hydro_inflows.py b/scripts/clean_tyndp_hydro_inflows.py index c90e8f564e..b35c47d791 100644 --- a/scripts/clean_tyndp_hydro_inflows.py +++ b/scripts/clean_tyndp_hydro_inflows.py @@ -47,14 +47,6 @@ def read_hydro_inflows_file( if not os.path.isfile(fn): return None - tech_res = { - "Run of River": "d", - "Pondage": "d", - "Reservoir": "w", - "PS Open": "w", - "PS Closed": "w", - } - inflow_tech = pd.read_excel( fn, skiprows=1, @@ -66,6 +58,9 @@ def read_hydro_inflows_file( sheet_name=f"{hydro_tech} - Year Dependent", ) + # infer resolution of data for each technology + tech_res = "w" if "Week" in inflow_tech.columns else "d" + sns_year = sns[0].year date_index = { "w": pd.date_range( @@ -83,18 +78,18 @@ def read_hydro_inflows_file( inflow_tech = ( inflow_tech.query("ShortName == 'INFLOW'") .assign( - datetime=date_index[tech_res[hydro_tech]], + datetime=date_index[tech_res], p_nom=lambda df: np.where( # calculate hourly inflow in GWh/h df.Variable.str.contains("week"), - df[int(cyear)].div(24 * 7), # the value will be either in GWh/week - df[int(cyear)].div(24), # the value will be either in GWh/day + df[int(cyear)].div(24 * 7), # input value was either in GWh/week + df[int(cyear)].div(24), # or in GWh/day ), ) .set_index("datetime") .reindex(sns) # filter for snapshots only .ffill() - .div(1e3) # convert from GW to MW .p_nom.rename(node) + .mul(1e3) # convert from GW to MW ) return inflow_tech From 277f0e2822eb80768ed424305ed70550cda91247 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 25 Jul 2025 12:04:10 +0200 Subject: [PATCH 03/25] feat: add more generalised retrieve for additional tyndp datasets and include rule for hydro inflows data --- rules/retrieve.smk | 16 +++++++++- ...a.py => retrieve_additional_tyndp_data.py} | 29 ++++++++++++------- 2 files changed, 34 insertions(+), 11 deletions(-) rename scripts/{retrieve_tyndp_pecd_data.py => retrieve_additional_tyndp_data.py} (53%) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 514551171c..523999c49a 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -190,13 +190,27 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" params: # TODO Integrate into Zenodo tyndp data bundle tyndp_bundle="data/tyndp_2024_bundle", + url="https://storage.googleapis.com/open-tyndp-data-store/PECD.zip", output: dir=directory("data/tyndp_2024_bundle/PECD"), log: "logs/retrieve_tyndp_pecd_data.log", retries: 2 script: - "../scripts/retrieve_tyndp_pecd_data.py" + "../scripts/retrieve_additional_tyndp_data.py" + + rule retrieve_tyndp_hydro_inflows: + params: + # TODO Integrate into Zenodo tyndp data bundle + tyndp_bundle="data/tyndp_2024_bundle", + url="https://storage.googleapis.com/open-tyndp-data-store/Hydro_Inflows.zip", + output: + dir=directory("data/tyndp_2024_bundle/Hydro Inflows"), + log: + "logs/retrieve_tyndp_hydro_inflows.log", + retries: 2 + script: + "../scripts/retrieve_additional_tyndp_data.py" ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_additional_tyndp_data.py similarity index 53% rename from scripts/retrieve_tyndp_pecd_data.py rename to scripts/retrieve_additional_tyndp_data.py index 8dc9ffdb1b..2d0ecfbe0d 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_additional_tyndp_data.py @@ -2,14 +2,22 @@ # # SPDX-License-Identifier: MIT """ -The TYNDP PECD data contains input data for the 2024 TYNDP scenario building process. - -This rule downloads the TYNDP PECD v3.1 data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` +Retrieves additional TYNDP data not included in the Zenodo TYNDP data bundle . +Downloads the zip file and extracts it in the ``data/tyndp_2024_bundle`` subdirectory, such that all files of the TYNDP bundle are stored in it. +Currently, this is used for two additional datasets: +* TYNDP PECD data: The TYNDP PECD v3.1 data contains input data for the 2024 TYNDP scenario building process. +* TYNDP hydro inflows: The TYNDP hydro inflow data from PEMMDB v2.4 contains hydro inflow data for different hydro technologies: + * Run of River, + * Pondage, + * Reservoir, + * PS Open, + * PS Closed + **Outputs** -- ``data/tyndp_2024_bundle/PECD``: PECD input data for TYNDP 2024 scenario building +- ``data/tyndp_2024_bundle/``: Additional input dataset for TYNDP 2024 scenario building """ @@ -22,9 +30,7 @@ logger = logging.getLogger(__name__) -# Define the base URL -# TODO: retrieve_tyndp_pecd_data needs to be deprecated once PECD data is added to the TYNDP data bundle -url = "https://storage.googleapis.com/open-tyndp-data-store/PECD.zip" +# TODO: retrieve_additional_tyndp_data needs to be deprecated once all TYNDP data is added to the TYNDP data bundle if __name__ == "__main__": if "snakemake" not in globals(): @@ -43,16 +49,19 @@ tyndp_bundle_fn = Path(rootpath, snakemake.params["tyndp_bundle"]) to_fn_zp = to_fn + ".zip" + # define url + url = snakemake.params.url + # download .zip file - logger.info(f"Downloading TYNDP PECD data from '{url}'.") + logger.info(f"Downloading additional TYNDP data from '{url}'.") progress_retrieve(url, to_fn_zp, disable=disable_progress) # extract - logger.info("Extracting TYNDP PECD data.") + logger.info("Extracting additional TYNDP data.") with zipfile.ZipFile(to_fn_zp, "r") as zip_ref: zip_ref.extractall(tyndp_bundle_fn) # remove .zip file os.remove(to_fn_zp) - logger.info(f"TYNDP PECD data available in '{to_fn}'.") + logger.info(f"Additional TYNDP data available in '{to_fn}'.") From cfcd6f6ce98582b809993f6a61ac8d91b81392dd Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 30 Jul 2025 17:27:56 +0200 Subject: [PATCH 04/25] feat: introduce safe_pyear helper function --- config/config.tyndp.yaml | 6 +++++ config/test/config.tyndp.yaml | 10 +++++++ rules/build_electricity.smk | 27 ++++++++++++++++--- scripts/_helpers.py | 33 ++++++++++++++++++++++++ scripts/build_renewable_profiles_pecd.py | 12 ++++----- scripts/clean_pecd_data.py | 9 +++++-- scripts/clean_tyndp_demand.py | 22 +++++++++------- 7 files changed, 98 insertions(+), 21 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index ca3fb41625..2e51bea53a 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -42,6 +42,12 @@ electricity: pecd_renewable_profiles: enable: true + fill_gaps_method: zero + # Complete PECD data is only available for the years 2030, 2040 + # TODO: adjust once udpated 2050 PECD data ist available + available_years: + - 2030 + - 2040 technologies: Wind_Offshore: - offwind-ac-fb-r diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index bfc66b1777..de1627e3df 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -53,6 +53,12 @@ electricity: pecd_renewable_profiles: enable: true + fill_gaps_method: zero + # Complete PECD data is only available for the years 2030, 2040 + # TODO: adjust once udpated 2050 PECD data ist available + available_years: + - 2030 + - 2040 technologies: Wind_Offshore: - offwind-ac-fb-r @@ -100,6 +106,10 @@ transmission_projects: load: source: tyndp # opsd, tyndp + available_years_tyndp: + - 2030 + - 2040 + - 2050 fill_gaps: enable: false manual_adjustments: false diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 9a82a789a8..d2518cc0b2 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -2,6 +2,8 @@ # # SPDX-License-Identifier: MIT +from scripts._helpers import safe_pyear + def input_elec_demand(w): return { @@ -397,6 +399,12 @@ rule clean_pecd_data: params: snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), + fill_gaps_method=config_provider( + "electricity", "pecd_renewable_profiles", "fill_gaps_method" + ), + available_years=config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + ), input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), @@ -420,16 +428,26 @@ def input_data_pecd(w): return { f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv") for pyear in set( - config_provider("scenario", "planning_horizons")(w) - ).intersection([2030, 2040]) - # Complete PECD data is only available for the years 2030, 2040 - # TODO: adjust if udpated 2050 data available + [ + safe_pyear( + year, + config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + )(w), + "PECD", + ) + for year in config_provider("scenario", "planning_horizons")(w) + ] + ) } rule build_renewable_profiles_pecd: params: planning_horizons=config_provider("scenario", "planning_horizons"), + available_years=config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + ), input: unpack(input_data_pecd), output: @@ -1083,6 +1101,7 @@ if config["load"]["source"] == "tyndp": planning_horizons=config_provider("scenario", "planning_horizons"), snapshots=config_provider("snapshots"), scenario=config_provider("tyndp_scenario"), + available_years=config_provider("load", "available_years_tyndp"), input: electricity_demand=directory("data/tyndp_2024_bundle/Demand Profiles"), output: diff --git a/scripts/_helpers.py b/scripts/_helpers.py index f1538ccca4..55ede2690d 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1129,3 +1129,36 @@ def extract_grid_data_tyndp( links.index = links.apply(make_index, axis=1, prefix=carrier) return links + + +def safe_pyear(year: int, available_years: list = [2030, 2040, 2050], source="TYNDP"): + """ + Checks and adjusts whether a given pyear is in the available years and falls back to the previous available year. + + Parameters + ---------- + year : int + planning horizon year which will be checked and possibly adjusted to previous available year + available_years : list + list of available years + source : str, optional + source of the data for which availability will be checked. Defaults to "TYNDP" + + Returns + ------- + year_new : str + safe pyear as a string + """ + + if not available_years: + raise ValueError("`available_years` cannot be empty.") + if year not in available_years: + lower = [y for y in available_years if y < year] + year_new = max(lower) if lower else available_years[0] + logger.warning( + f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}." + ) + else: + year_new = year + + return str(year_new) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 67646349b6..f4dcd5d3c8 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -22,12 +22,12 @@ import logging -import numpy as np import pandas as pd import xarray as xr from scripts._helpers import ( configure_logging, + safe_pyear, set_scenario_config, ) @@ -55,11 +55,11 @@ f"Extract PECD capacity factor time series for year {year} for technology {technology}..." ) year_i = year - if int(year) not in [2030, 2040, 2050]: - year = np.clip(10 * (year // 10), 2030, 2050) - logger.warning( - f"TYNDP PECD data unavailable for planning horizon. Falling back to previous available year {year}." - ) + # falling back to latest available pyear if not in list of available years + year = safe_pyear( + int(year), available_years=snakemake.params.available_years, source="PECD" + ) + # TODO: remove once PECD data is updated if year == 2050: logger.warning( "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data." diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index c582b08b1d..694e134c0d 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -30,6 +30,7 @@ from scripts._helpers import ( configure_logging, get_snapshots, + safe_pyear, set_scenario_config, ) @@ -94,8 +95,12 @@ def read_pecd_file( ) cyear = 2009 - # Planning year - pyear = str(snakemake.wildcards.planning_horizons) + # Planning year (falls back to latest available pyear if not in list of available years) + pyear = safe_pyear( + int(snakemake.wildcards.planning_horizons), + available_years=snakemake.params.available_years, + source="PECD", + ) # TODO: find solution for solar profiles being differentiated between Utility and Rooftop for Italy # Technology as in PECD terminology diff --git a/scripts/clean_tyndp_demand.py b/scripts/clean_tyndp_demand.py index 3a03c391f5..72d34f2a51 100644 --- a/scripts/clean_tyndp_demand.py +++ b/scripts/clean_tyndp_demand.py @@ -12,15 +12,21 @@ from functools import partial from pathlib import Path -import numpy as np import pandas as pd -from _helpers import configure_logging, get_snapshots, set_scenario_config +from _helpers import ( + configure_logging, + get_snapshots, + safe_pyear, + set_scenario_config, +) from tqdm import tqdm logger = logging.getLogger(__name__) -def load_elec_demand(fn: str, scenario: str, pyear: int, cyear: int): +def load_elec_demand( + fn: str, scenario: str, pyear: int, cyear: int, available_years: list +): """ Load electricity demand files into dictionary of dataframes. Filter for specific climatic year and format data. """ @@ -28,12 +34,9 @@ def load_elec_demand(fn: str, scenario: str, pyear: int, cyear: int): # handle intermediate years # TODO: Possibly improve this with linear interpolation for 2035 and 2045 - if pyear not in [2030, 2040, 2050]: - pyear = np.clip(10 * (pyear // 10), 2030, 2050) - logger.warning( - "Planning horizon doesn't match available 2024 TYNDP electricity demand data. " - f"Falling back to previous available year {pyear}." - ) + pyear = int( + safe_pyear(int(pyear), available_years=available_years, source="TYNDP demand") + ) if scenario == "NT": if pyear == 2050: logger.warning( @@ -132,6 +135,7 @@ def load_elec_demand(fn: str, scenario: str, pyear: int, cyear: int): snakemake.input.electricity_demand, scenario, cyear=cyear, + available_years=snakemake.params.available_years, ) with mp.Pool(processes=snakemake.threads) as pool: From 9bf086b98d5869199fc7a01d4346ac7bfabe869f Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 31 Jul 2025 14:27:54 +0200 Subject: [PATCH 05/25] feat: add verbose option to safe_pyear helper function --- rules/build_electricity.smk | 1 + scripts/_helpers.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index d2518cc0b2..2ceffed119 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -435,6 +435,7 @@ def input_data_pecd(w): "electricity", "pecd_renewable_profiles", "available_years" )(w), "PECD", + verbose=False, ) for year in config_provider("scenario", "planning_horizons")(w) ] diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 55ede2690d..16936bd86d 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1131,7 +1131,12 @@ def extract_grid_data_tyndp( return links -def safe_pyear(year: int, available_years: list = [2030, 2040, 2050], source="TYNDP"): +def safe_pyear( + year: int, + available_years: list = [2030, 2040, 2050], + source: str = "TYNDP", + verbose: bool = True, +): """ Checks and adjusts whether a given pyear is in the available years and falls back to the previous available year. @@ -1139,10 +1144,12 @@ def safe_pyear(year: int, available_years: list = [2030, 2040, 2050], source="TY ---------- year : int planning horizon year which will be checked and possibly adjusted to previous available year - available_years : list - list of available years + available_years : list, optional + list of available years. Defaults to [2030, 2040, 2050] source : str, optional source of the data for which availability will be checked. Defaults to "TYNDP" + verbose : bool, optional + Whether to activate verbose logging. Defaults to True Returns ------- @@ -1155,9 +1162,10 @@ def safe_pyear(year: int, available_years: list = [2030, 2040, 2050], source="TY if year not in available_years: lower = [y for y in available_years if y < year] year_new = max(lower) if lower else available_years[0] - logger.warning( - f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}." - ) + if verbose: + logger.warning( + f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}." + ) else: year_new = year From 4a2a6d01e252a9c77b2648eb899dd9b925529c3d Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 31 Jul 2025 18:05:36 +0200 Subject: [PATCH 06/25] feat: add config section with switch, techs and available years --- config/config.default.yaml | 13 +++++++++++++ config/config.tyndp.yaml | 13 +++++++++++++ config/test/config.tyndp.yaml | 13 +++++++++++++ rules/build_electricity.smk | 19 +++++++++++++++---- rules/build_sector.smk | 3 +-- 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 5f046bbc23..f76c38c790 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -155,6 +155,19 @@ electricity: - offwind-h2-fb-oh - offwind-h2-fl-oh + tyndp_hydro_profiles: + enable: true + technologies: + - Run of River + - Pondage + - Reservoir + - PS Open + - PS Closed + available_years: + - 2030 + - 2040 + - 2050 + estimate_renewable_capacities: enable: true from_gem: true diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 2e51bea53a..c27dfda4d8 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -59,6 +59,19 @@ electricity: - offwind-h2-fb-oh - offwind-h2-fl-oh + tyndp_hydro_profiles: + enable: true + technologies: + - Run of River + - Pondage + - Reservoir + - PS Open + - PS Closed + available_years: + - 2030 + - 2040 + - 2050 + estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation technologies: diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index de1627e3df..74b680c2ef 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -70,6 +70,19 @@ electricity: - offwind-h2-fb-oh - offwind-h2-fl-oh + tyndp_hydro_profiles: + enable: true + technologies: + - Run of River + - Pondage + - Reservoir + - PS Open + - PS Closed + available_years: + - 2030 + - 2040 + - 2050 + estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation technologies: diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 2ceffed119..3f599b6f5b 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -544,10 +544,21 @@ def input_data_hydro_tyndp(w): f"hydro_inflows_tyndp_{tech}_{str(pyear)}.csv" ) for pyear in set( - config_provider("scenario", "planning_horizons")(w) - ).intersection([2030, 2040, 2050]) - # Hydro inflows data is only available for the years 2030, 2040, 2050 - for tech in ["Run of River", "Pondage", "Reservoir", "PS Open", "PS Closed"] + [ + safe_pyear( + year, + config_provider( + "electricity", "tyndp_hydro_profiles", "available_years" + )(w), + "PEMMDB hydro", + verbose=False, + ) + for year in config_provider("scenario", "planning_horizons")(w) + ] + ) + for tech in config_provider( + "electricity", "tyndp_hydro_profiles", "technologies" + )(w) } diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 432bb80ada..404d09eeaa 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1573,8 +1573,7 @@ rule prepare_sector_network: else [] ), profile_hydro_tyndp=branch( - lambda w: "hydro" - in config_provider("electricity", "tyndp_renewable_carriers")(w), + config_provider("electricity", "tyndp_hydro_profiles", "enable"), resources("profile_hydro_tyndp.nc"), [], ), From 9299cec28b1950d4983ca13b4f352b868fd7b10a Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 1 Aug 2025 09:57:57 +0200 Subject: [PATCH 07/25] feat: build hydro profile file with dimensions bus, time, year, hydro_tech --- rules/build_electricity.smk | 10 +++++ scripts/build_tyndp_hydro_profile.py | 67 +++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 3f599b6f5b..f14b148859 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -563,6 +563,16 @@ def input_data_hydro_tyndp(w): rule build_tyndp_hydro_profile: + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + planning_horizons=config_provider("scenario", "planning_horizons"), + available_years=config_provider( + "electricity", "tyndp_hydro_profiles", "available_years" + ), + technologies=config_provider( + "electricity", "tyndp_hydro_profiles", "technologies" + ), input: unpack(input_data_hydro_tyndp), output: diff --git a/scripts/build_tyndp_hydro_profile.py b/scripts/build_tyndp_hydro_profile.py index 8151e12bd3..e42e7588ca 100644 --- a/scripts/build_tyndp_hydro_profile.py +++ b/scripts/build_tyndp_hydro_profile.py @@ -12,7 +12,72 @@ =================== ================ ========================================================= Field Dimensions Description =================== ================ ========================================================= - inflow countries, time, Inflow to the state of charge (in MW), + inflow bus, time, Inflow to the state of charge (in MW), year, hydro_tech e.g. due to river inflow in hydro reservoir. =================== ================ ========================================================= """ + +import logging + +import pandas as pd +import xarray as xr +from tqdm.contrib.itertools import product + +from scripts._helpers import ( + configure_logging, + get_snapshots, + safe_pyear, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake("build_tyndp_hydro_profile") + configure_logging(snakemake) + set_scenario_config(snakemake) + + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + + years_in_time = pd.DatetimeIndex(time).year.unique() + + technologies = snakemake.params.technologies + pyears = snakemake.params.planning_horizons + + inflows = [] + + for year, technology in product(pyears, technologies): + logger.info( + f"Extracting hydro inflows for year {year} for technology {technology}..." + ) + year_i = year + # falling back to latest available pyear if not in list of available years + year = safe_pyear( + int(year), + available_years=snakemake.params.available_years, + source="PEMMDB hydro inflow", + ) + + inflow = ( + pd.read_csv( + snakemake.input[f"hydro_inflow_tyndp_{technology}_{year}"], + parse_dates=True, + index_col=0, + ) + .rename_axis("time") + .reset_index() + .melt(id_vars=["time"], var_name="bus", value_name="profile") + .assign(year=year_i, hydro_tech=technology) + .set_index(["bus", "time", "year", "hydro_tech"]) + .to_xarray() + ) + + inflows.append(inflow) + + ds = xr.merge(inflows) + ds = ds.sel(time=time) + + ds.to_netcdf(snakemake.output.profile) From 0165421474a82eae11d0c8d064a9077da0b212a5 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 1 Aug 2025 09:58:43 +0200 Subject: [PATCH 08/25] feat: add hydro to tyndp test config --- config/test/config.tyndp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 74b680c2ef..915a92f3c6 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -49,7 +49,7 @@ electricity: Link: [H2 pipeline] renewable_carriers: [solar, solar-hsat, onwind] - tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + tyndp_renewable_carriers: [hydro, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true From 2cec73804322a7fbad6b5b8e4017112aa5ec66d0 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 1 Aug 2025 10:15:26 +0200 Subject: [PATCH 09/25] fix: disable tyndp_hydro_profiles in default config --- config/config.default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index f76c38c790..0aa59abc88 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -156,7 +156,7 @@ electricity: - offwind-h2-fl-oh tyndp_hydro_profiles: - enable: true + enable: false technologies: - Run of River - Pondage From bb4991931cf707f3c208caa60d0e10522d4bff31 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 1 Aug 2025 11:11:38 +0200 Subject: [PATCH 10/25] fix: differentiate hydro profiles in prepare_sector_network.py --- scripts/prepare_sector_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index e6c176a294..84c75d229b 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -7508,7 +7508,7 @@ def add_import_options( profiles = { key: snakemake.input[key] for key in snakemake.input.keys() - if key.startswith("profile") + if key.startswith("profile") and "hydro" not in key } pecd_renewable_profiles_techs = snakemake.params.electricity[ "pecd_renewable_profiles" From f0427b893df297cf802bd8bf5c1ded196fa8f612 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 1 Aug 2025 17:48:28 +0200 Subject: [PATCH 11/25] doc: document new config option --- doc/configtables/electricity.csv | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 2e8a61e495..605fb6920a 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -36,6 +36,10 @@ pecd_renewable_profiles,,, -- technologies,,, -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles. +tyndp_hydro_profiles,,, +-- enable,,bool,Activate PEMMDB hyddro inflow profiles from 2024 TYNDP instead of default hydro profiles. +-- available_years,--,list,"List of years for which PEMMDB hydro inflows data is available." +-- technologies,--,list,The hydro technology for which PEMMDB hydro inflows data shall be prepared. Technologies can be any of {Run of River, Pondage, Reservoir, PS Open, PS Closed}. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. From 0ceee6e74146afb2a4d0cfa7df1152c370f0604b Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 1 Aug 2025 17:56:19 +0200 Subject: [PATCH 12/25] doc: add release note --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 68a3873d2c..b5594f46fd 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -11,7 +11,7 @@ Release Notes Upcoming Open-TYNDP Release ================ - +* Introduce processing of PEMMDB hydro inflows data for different hydro technologies (Run of River, Pondage, Reservoir, PS Open, PS Closed) from 2024 TYNDP into hydro inflow profiles (https://github.com/open-energy-transition/open-tyndp/pull/77). This implementation serves to facilitate a sub-workflow for creation of the hydro inflow profiles, but does not yet attach them to any hydro technologies. Upcoming PyPSA-Eur Release From 9fbf3b26ba4d8b9545112a5152db0c98719b05ec Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 12 Aug 2025 09:42:53 +0200 Subject: [PATCH 13/25] ci: add explicit ruleorder for tyndp hydro rules --- rules/retrieve.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 36ce4e5361..f3d49f0d6e 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -217,6 +217,7 @@ if config["enable"]["retrieve"]: "../scripts/retrieve_additional_tyndp_data.py" ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data + ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_hydro_inflows > clean_tyndp_hydro_inflows rule retrieve_countries_centroids: output: From 9faaccc6e295a28cc4ce4f1b864a8c438b2d077a Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 12 Aug 2025 09:52:20 +0200 Subject: [PATCH 14/25] doc: add documentation for retrieve_tyndp_hydro_inflows rule --- doc/retrieve.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/retrieve.rst b/doc/retrieve.rst index b5eb5d65bc..94624e7d82 100644 --- a/doc/retrieve.rst +++ b/doc/retrieve.rst @@ -172,4 +172,9 @@ None. Rule ``retrieve_tyndp_pecd_data`` ==================================== -.. automodule:: retrieve_tyndp_pecd_data +.. automodule:: retrieve_additional_tyndp_data + +Rule ``retrieve_tyndp_hydro_inflows`` +==================================== + +.. automodule:: retrieve_additional_tyndp_data From 9abc99d84469ed6f97c0b1d3f8515e99eaaee036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20R=C3=BCdt?= <117752024+daniel-rdt@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:34:32 +0200 Subject: [PATCH 15/25] Apply suggestions from code review Co-authored-by: Thomas Gilon --- doc/configtables/electricity.csv | 4 ++-- doc/release_notes.rst | 2 +- scripts/build_tyndp_hydro_profile.py | 6 +++--- scripts/clean_tyndp_hydro_inflows.py | 4 ++-- scripts/retrieve_additional_tyndp_data.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index cb0185770a..b8679cc445 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -39,9 +39,9 @@ pecd_renewable_profiles,,, -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles. tyndp_hydro_profiles,,, --- enable,,bool,Activate PEMMDB hyddro inflow profiles from 2024 TYNDP instead of default hydro profiles. +-- enable,,bool,Activate PEMMDB hydro inflow profiles from 2024 TYNDP instead of default hydro profiles. -- available_years,--,list,"List of years for which PEMMDB hydro inflows data is available." --- technologies,--,list,The hydro technology for which PEMMDB hydro inflows data shall be prepared. Technologies can be any of {Run of River, Pondage, Reservoir, PS Open, PS Closed}. +-- technologies,--,list,The hydro technologies for which PEMMDB hydro inflows data is used. Technologies can be any of {Run of River, Pondage, Reservoir, PS Open, PS Closed}. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 595249bd77..f3538863a6 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -14,7 +14,7 @@ Upcoming Open-TYNDP Release * Add complete compatibility for processing and preparation of PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP) (https://github.com/open-energy-transition/open-tyndp/pull/71). These profiles are used for the TYNDP 2024 and replace the default ERA5- and SARAH3-based profiles processed with Atlite. This implementation serves to facilitate a sub-workflow for creation of the renewable profiles, but does not yet attach them to any technologies. -* Introduce processing of PEMMDB hydro inflows data for different hydro technologies (Run of River, Pondage, Reservoir, PS Open, PS Closed) from 2024 TYNDP into hydro inflow profiles (https://github.com/open-energy-transition/open-tyndp/pull/77). This implementation serves to facilitate a sub-workflow for creation of the hydro inflow profiles, but does not yet attach them to any hydro technologies. +* Introduce processing of PEMMDB hydro inflows data for different hydro technologies (Run of River, Pondage, Reservoir, PS Open, PS Closed) from the 2024 TYNDP to create hydro inflow profiles (https://github.com/open-energy-transition/open-tyndp/pull/77). This implementation facilitates the sub-workflow for creating the hydro inflow profiles, but it does not yet attach them to any hydro technologies. Upcoming PyPSA-Eur Release diff --git a/scripts/build_tyndp_hydro_profile.py b/scripts/build_tyndp_hydro_profile.py index e42e7588ca..6219a56162 100644 --- a/scripts/build_tyndp_hydro_profile.py +++ b/scripts/build_tyndp_hydro_profile.py @@ -1,8 +1,8 @@ -# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# SPDX-FileCopyrightText: Contributors to Open-TYNDP # # SPDX-License-Identifier: MIT """ -Build hydroelectric inflow time-series for each country based on TYNDP hydro inflow data. +Builds hydroelectric inflow time-series for each country based on TYNDP hydro inflow data. Outputs ------- @@ -51,7 +51,7 @@ for year, technology in product(pyears, technologies): logger.info( - f"Extracting hydro inflows for year {year} for technology {technology}..." + f"Extracting hydro inflows for {technology} in {year}" ) year_i = year # falling back to latest available pyear if not in list of available years diff --git a/scripts/clean_tyndp_hydro_inflows.py b/scripts/clean_tyndp_hydro_inflows.py index b35c47d791..53226e8003 100644 --- a/scripts/clean_tyndp_hydro_inflows.py +++ b/scripts/clean_tyndp_hydro_inflows.py @@ -113,7 +113,7 @@ def read_hydro_inflows_file( cyear = sns[0].year if int(cyear) < 1982 or int(cyear) > 2019: logger.warning( - "Snapshot year doesn't match available TYNDP data. Falling back to 2009." + f"Snapshot year {cyear} doesn't match available TYNDP data. Falling back to 2009." ) cyear = 2009 @@ -121,8 +121,8 @@ def read_hydro_inflows_file( pyear = str(snakemake.wildcards.planning_horizons) hydro_tech = str(snakemake.wildcards.tech) + # Parameters onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) - nodes = onshore_buses.index.str.replace("GB", "UK", regex=True) hydro_inflows_dir = snakemake.input.hydro_inflows_dir diff --git a/scripts/retrieve_additional_tyndp_data.py b/scripts/retrieve_additional_tyndp_data.py index d9fba4f0a6..8c6f29e06b 100644 --- a/scripts/retrieve_additional_tyndp_data.py +++ b/scripts/retrieve_additional_tyndp_data.py @@ -2,8 +2,8 @@ # # SPDX-License-Identifier: MIT """ -Retrieves additional TYNDP data not included in the Zenodo TYNDP data bundle . -Downloads the zip file and extracts it in the ``data/tyndp_2024_bundle`` +Retrieves additional TYNDP data not included in the Zenodo TYNDP data bundle. +Downloads the zip file from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` subdirectory, such that all files of the TYNDP bundle are stored in it. The original data is published by ENTSO-E and ENTSOG under Creative Commons Attribution 4.0 International License (CC-BY 4.0) and can be found under https://2024.entsos-tyndp-scenarios.eu/download/. From 45d6b3b70074c7b5382afcb03a7b7797548ec294 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 14:36:18 +0000 Subject: [PATCH 16/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_tyndp_hydro_profile.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/build_tyndp_hydro_profile.py b/scripts/build_tyndp_hydro_profile.py index 6219a56162..519141db8c 100644 --- a/scripts/build_tyndp_hydro_profile.py +++ b/scripts/build_tyndp_hydro_profile.py @@ -50,9 +50,7 @@ inflows = [] for year, technology in product(pyears, technologies): - logger.info( - f"Extracting hydro inflows for {technology} in {year}" - ) + logger.info(f"Extracting hydro inflows for {technology} in {year}") year_i = year # falling back to latest available pyear if not in list of available years year = safe_pyear( From c6259e1b90885d55ed46aa4d576b46eec1e46e9c Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 14 Aug 2025 16:42:23 +0200 Subject: [PATCH 17/25] refactor: change the order of config parameters for consistency --- config/config.default.yaml | 8 ++++---- config/config.tyndp.yaml | 8 ++++---- config/test/config.tyndp.yaml | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 3f7c3a4791..64f16081f3 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -163,16 +163,16 @@ electricity: tyndp_hydro_profiles: enable: false + available_years: + - 2030 + - 2040 + - 2050 technologies: - Run of River - Pondage - Reservoir - PS Open - PS Closed - available_years: - - 2030 - - 2040 - - 2050 estimate_renewable_capacities: enable: true diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 194d874f0a..147eee5b09 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -68,16 +68,16 @@ electricity: tyndp_hydro_profiles: enable: true + available_years: + - 2030 + - 2040 + - 2050 technologies: - Run of River - Pondage - Reservoir - PS Open - PS Closed - available_years: - - 2030 - - 2040 - - 2050 estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index a831641eed..15746279f9 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -80,16 +80,16 @@ electricity: tyndp_hydro_profiles: enable: true + available_years: + - 2030 + - 2040 + - 2050 technologies: - Run of River - Pondage - Reservoir - PS Open - PS Closed - available_years: - - 2030 - - 2040 - - 2050 estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation From 465be656e29d3db1a8c7dd3eba37d3c89fccd8c8 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 14 Aug 2025 17:42:19 +0200 Subject: [PATCH 18/25] refactor: rename profile_hydro_tyndp to profile_pemmdb_hydro --- config/config.default.yaml | 2 +- config/config.tyndp.yaml | 2 +- config/test/config.tyndp.yaml | 2 +- doc/configtables/electricity.csv | 2 +- rules/build_electricity.smk | 10 +++++----- rules/build_sector.smk | 6 +++--- scripts/build_tyndp_hydro_profile.py | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 64f16081f3..88becba3c5 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -161,7 +161,7 @@ electricity: - offwind-h2-fb-oh - offwind-h2-fl-oh - tyndp_hydro_profiles: + pemmdb_hydro_profiles: enable: false available_years: - 2030 diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 147eee5b09..ec47ea3f05 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -66,7 +66,7 @@ electricity: Wind_Onshore: - onwind - tyndp_hydro_profiles: + pemmdb_hydro_profiles: enable: true available_years: - 2030 diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 15746279f9..7823d8dd65 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -78,7 +78,7 @@ electricity: Wind_Onshore: - onwind - tyndp_hydro_profiles: + pemmdb_hydro_profiles: enable: true available_years: - 2030 diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index b8679cc445..e08c750da9 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -38,7 +38,7 @@ pecd_renewable_profiles,,, -- technologies,,, -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles. -tyndp_hydro_profiles,,, +pemmdb_hydro_profiles,,, -- enable,,bool,Activate PEMMDB hydro inflow profiles from 2024 TYNDP instead of default hydro profiles. -- available_years,--,list,"List of years for which PEMMDB hydro inflows data is available." -- technologies,--,list,The hydro technologies for which PEMMDB hydro inflows data is used. Technologies can be any of {Run of River, Pondage, Reservoir, PS Open, PS Closed}. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 76596e142b..05a206a30a 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -545,7 +545,7 @@ def input_data_hydro_tyndp(w): safe_pyear( year, config_provider( - "electricity", "tyndp_hydro_profiles", "available_years" + "electricity", "pemmdb_hydro_profiles", "available_years" )(w), "PEMMDB hydro", verbose=False, @@ -554,7 +554,7 @@ def input_data_hydro_tyndp(w): ] ) for tech in config_provider( - "electricity", "tyndp_hydro_profiles", "technologies" + "electricity", "pemmdb_hydro_profiles", "technologies" )(w) } @@ -565,15 +565,15 @@ rule build_tyndp_hydro_profile: drop_leap_day=config_provider("enable", "drop_leap_day"), planning_horizons=config_provider("scenario", "planning_horizons"), available_years=config_provider( - "electricity", "tyndp_hydro_profiles", "available_years" + "electricity", "pemmdb_hydro_profiles", "available_years" ), technologies=config_provider( - "electricity", "tyndp_hydro_profiles", "technologies" + "electricity", "pemmdb_hydro_profiles", "technologies" ), input: unpack(input_data_hydro_tyndp), output: - profile=resources("profile_hydro_tyndp.nc"), + profile=resources("profile_pemmdb_hydro.nc"), log: logs("build_tyndp_hydro_profile.log"), benchmark: diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 2f83f2edd2..9153d567cd 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1573,9 +1573,9 @@ rule prepare_sector_network: if config_provider("sector", "h2_topology_tyndp")(w) else [] ), - profile_hydro_tyndp=branch( - config_provider("electricity", "tyndp_hydro_profiles", "enable"), - resources("profile_hydro_tyndp.nc"), + profile_pemmdb_hydro=branch( + config_provider("electricity", "pemmdb_hydro_profiles", "enable"), + resources("profile_pemmdb_hydro.nc"), [], ), output: diff --git a/scripts/build_tyndp_hydro_profile.py b/scripts/build_tyndp_hydro_profile.py index 519141db8c..0d85f83f5d 100644 --- a/scripts/build_tyndp_hydro_profile.py +++ b/scripts/build_tyndp_hydro_profile.py @@ -2,12 +2,12 @@ # # SPDX-License-Identifier: MIT """ -Builds hydroelectric inflow time-series for each country based on TYNDP hydro inflow data. +Builds hydroelectric inflow time-series for each country based on PEMMDB v2.4 hydro inflow data from the 2024 TYNDP. Outputs ------- -- ``resources/profile_hydro_tyndp.nc``: +- ``resources/profile_pemmdb_hydro.nc``: =================== ================ ========================================================= Field Dimensions Description From fb30763c14e1b540e7dc996c95e43a53a5450dae Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 14 Aug 2025 17:55:37 +0200 Subject: [PATCH 19/25] apply more suggestions from code review --- rules/build_electricity.smk | 2 +- rules/retrieve.smk | 6 +----- scripts/clean_tyndp_hydro_inflows.py | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 05a206a30a..16057dff8e 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -519,7 +519,7 @@ rule clean_tyndp_hydro_inflows: drop_leap_day=config_provider("enable", "drop_leap_day"), input: hydro_inflows_dir="data/tyndp_2024_bundle/Hydro Inflows", - onshore_buses=resources("busmap_base_s_all.csv"), + busmap=resources("busmap_base_s_all.csv"), output: hydro_inflows_tyndp=resources( "hydro_inflows_tyndp_{tech}_{planning_horizons}.csv" diff --git a/rules/retrieve.smk b/rules/retrieve.smk index f3d49f0d6e..eb61ee1ba1 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -203,18 +203,14 @@ if config["enable"]["retrieve"]: script: "../scripts/retrieve_additional_tyndp_data.py" - rule retrieve_tyndp_hydro_inflows: + use rule retrieve_tyndp_pecd_data as retrieve_tyndp_hydro_inflows with: params: # TODO Integrate into Zenodo tyndp data bundle - tyndp_bundle="data/tyndp_2024_bundle", url="https://storage.googleapis.com/open-tyndp-data-store/Hydro_Inflows.zip", output: dir=directory("data/tyndp_2024_bundle/Hydro Inflows"), log: "logs/retrieve_tyndp_hydro_inflows.log", - retries: 2 - script: - "../scripts/retrieve_additional_tyndp_data.py" ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_hydro_inflows > clean_tyndp_hydro_inflows diff --git a/scripts/clean_tyndp_hydro_inflows.py b/scripts/clean_tyndp_hydro_inflows.py index 53226e8003..a58c8929ac 100644 --- a/scripts/clean_tyndp_hydro_inflows.py +++ b/scripts/clean_tyndp_hydro_inflows.py @@ -41,7 +41,7 @@ def read_hydro_inflows_file( pyear: str, hydro_tech: str, sns: pd.DatetimeIndex, -): +) -> pd.Series: fn = Path(hydro_inflows_dir, pyear, f"PEMMDB_{node}_Hydro_Inflows_{pyear}.xlsx") if not os.path.isfile(fn): @@ -122,7 +122,7 @@ def read_hydro_inflows_file( hydro_tech = str(snakemake.wildcards.tech) # Parameters - onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) + onshore_buses = pd.read_csv(snakemake.input.busmap, index_col=0) nodes = onshore_buses.index.str.replace("GB", "UK", regex=True) hydro_inflows_dir = snakemake.input.hydro_inflows_dir From 01b079a17b3f529ea891c9af232ac11db80e7069 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 14 Aug 2025 18:02:47 +0200 Subject: [PATCH 20/25] doc: add autodoc for missing hydro profile and pecd rules --- doc/preparation.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/preparation.rst b/doc/preparation.rst index fc9f5a29e9..48fcdb0fac 100644 --- a/doc/preparation.rst +++ b/doc/preparation.rst @@ -208,3 +208,25 @@ Rule ``prepare_network`` .. automodule:: prepare_network +Rule ``clean_pecd_data`` +=========================== + +.. automodule:: clean_pecd_data + +Rule ``build_renewable_profiles_pecd`` +=========================== + +.. automodule:: build_renewable_profiles_pecd + +Rule ``clean_tyndp_hydro_inflows`` +=========================== + +.. automodule:: clean_tyndp_hydro_inflows + +Rule ``build_tyndp_hydro_profile`` +=========================== + +.. automodule:: build_tyndp_hydro_profile + + + From a75240211aa5705b660d416069259e7c3e14d348 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 14 Aug 2025 18:08:55 +0200 Subject: [PATCH 21/25] refactor: move static variable outside of function --- scripts/clean_tyndp_hydro_inflows.py | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/clean_tyndp_hydro_inflows.py b/scripts/clean_tyndp_hydro_inflows.py index a58c8929ac..4807f51b0a 100644 --- a/scripts/clean_tyndp_hydro_inflows.py +++ b/scripts/clean_tyndp_hydro_inflows.py @@ -41,6 +41,7 @@ def read_hydro_inflows_file( pyear: str, hydro_tech: str, sns: pd.DatetimeIndex, + date_index: dict, ) -> pd.Series: fn = Path(hydro_inflows_dir, pyear, f"PEMMDB_{node}_Hydro_Inflows_{pyear}.xlsx") @@ -61,20 +62,6 @@ def read_hydro_inflows_file( # infer resolution of data for each technology tech_res = "w" if "Week" in inflow_tech.columns else "d" - sns_year = sns[0].year - date_index = { - "w": pd.date_range( - start=f"{sns_year}-01-01", - periods=53, # 53 weeks - freq="7D", - ), - "d": pd.date_range( - start=f"{sns_year}-01-01", - periods=366, # 366 days (incl. first day of next year) - freq="D", - ), - } - inflow_tech = ( inflow_tech.query("ShortName == 'INFLOW'") .assign( @@ -111,6 +98,18 @@ def read_hydro_inflows_file( # Climate year from snapshots sns = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) cyear = sns[0].year + date_index = { + "w": pd.date_range( + start=f"{cyear}-01-01", + periods=53, # 53 weeks + freq="7D", + ), + "d": pd.date_range( + start=f"{cyear}-01-01", + periods=366, # 366 days (incl. first day of next year) + freq="D", + ), + } if int(cyear) < 1982 or int(cyear) > 2019: logger.warning( f"Snapshot year {cyear} doesn't match available TYNDP data. Falling back to 2009." @@ -119,12 +118,12 @@ def read_hydro_inflows_file( # Planning year pyear = str(snakemake.wildcards.planning_horizons) - hydro_tech = str(snakemake.wildcards.tech) # Parameters onshore_buses = pd.read_csv(snakemake.input.busmap, index_col=0) nodes = onshore_buses.index.str.replace("GB", "UK", regex=True) hydro_inflows_dir = snakemake.input.hydro_inflows_dir + hydro_tech = str(snakemake.wildcards.tech) # Load and prep inflow data tqdm_kwargs = { @@ -141,6 +140,7 @@ def read_hydro_inflows_file( pyear=pyear, hydro_tech=hydro_tech, sns=sns, + date_index=date_index, ) with mp.Pool(processes=snakemake.threads) as pool: From d3b0f8b92a38c3bbf4636b114d8c19090f8d70ad Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 14 Aug 2025 18:27:39 +0200 Subject: [PATCH 22/25] refactor: remove redundant fill value and clean up GB and UK renaming --- scripts/clean_tyndp_hydro_inflows.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/clean_tyndp_hydro_inflows.py b/scripts/clean_tyndp_hydro_inflows.py index 4807f51b0a..029ec6f92b 100644 --- a/scripts/clean_tyndp_hydro_inflows.py +++ b/scripts/clean_tyndp_hydro_inflows.py @@ -43,7 +43,11 @@ def read_hydro_inflows_file( sns: pd.DatetimeIndex, date_index: dict, ) -> pd.Series: - fn = Path(hydro_inflows_dir, pyear, f"PEMMDB_{node}_Hydro_Inflows_{pyear}.xlsx") + fn = Path( + hydro_inflows_dir, + pyear, + f"PEMMDB_{node.replace('GB', 'UK')}_Hydro_Inflows_{pyear}.xlsx", + ) if not os.path.isfile(fn): return None @@ -121,7 +125,7 @@ def read_hydro_inflows_file( # Parameters onshore_buses = pd.read_csv(snakemake.input.busmap, index_col=0) - nodes = onshore_buses.index.str.replace("GB", "UK", regex=True) + nodes = onshore_buses.index hydro_inflows_dir = snakemake.input.hydro_inflows_dir hydro_tech = str(snakemake.wildcards.tech) @@ -149,12 +153,10 @@ def read_hydro_inflows_file( inflows_df = ( pd.concat(inflows, axis=1) .reindex( - nodes, axis=1, fill_value=0.0 + nodes, + axis=1, ) # include missing node data with empty columns - .rename( - columns=lambda x: x.replace("UK", "GB") - ) # replace UK with GB for naming convention - .fillna(0.0) + .fillna(0.0) # fill missing data with zero values ) inflows_df.to_csv(snakemake.output.hydro_inflows_tyndp) From 1baf2590d430c32ac4f81802cacdde7a6cf37b66 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 14 Aug 2025 18:39:21 +0200 Subject: [PATCH 23/25] feat: add source parameter for logging of retrieve_additional_tyndp_data --- rules/retrieve.smk | 2 ++ scripts/retrieve_additional_tyndp_data.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index eb61ee1ba1..27e6265027 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -195,6 +195,7 @@ if config["enable"]["retrieve"]: # TODO Integrate into Zenodo tyndp data bundle tyndp_bundle="data/tyndp_2024_bundle", url="https://storage.googleapis.com/open-tyndp-data-store/PECD.zip", + source="PECD", output: dir=directory("data/tyndp_2024_bundle/PECD"), log: @@ -207,6 +208,7 @@ if config["enable"]["retrieve"]: params: # TODO Integrate into Zenodo tyndp data bundle url="https://storage.googleapis.com/open-tyndp-data-store/Hydro_Inflows.zip", + source="hydro inflows", output: dir=directory("data/tyndp_2024_bundle/Hydro Inflows"), log: diff --git a/scripts/retrieve_additional_tyndp_data.py b/scripts/retrieve_additional_tyndp_data.py index 8c6f29e06b..e9ccc49db6 100644 --- a/scripts/retrieve_additional_tyndp_data.py +++ b/scripts/retrieve_additional_tyndp_data.py @@ -47,6 +47,7 @@ set_scenario_config(snakemake) disable_progress = snakemake.config["run"].get("disable_progressbar", False) + source = snakemake.params.source to_fn = snakemake.output.dir tyndp_bundle_fn = Path(rootpath, snakemake.params["tyndp_bundle"]) to_fn_zp = to_fn + ".zip" @@ -55,15 +56,15 @@ url = snakemake.params.url # download .zip file - logger.info(f"Downloading additional TYNDP data from '{url}'.") + logger.info(f"Downloading TYNDP {source} data from '{url}'.") progress_retrieve(url, to_fn_zp, disable=disable_progress) # extract - logger.info("Extracting additional TYNDP data.") + logger.info(f"Extracting TYNDP {source} data.") with zipfile.ZipFile(to_fn_zp, "r") as zip_ref: zip_ref.extractall(tyndp_bundle_fn) # remove .zip file os.remove(to_fn_zp) - logger.info(f"Additional TYNDP data available in '{to_fn}'.") + logger.info(f"TYNDP {source} data available in '{to_fn}'.") From 0e07ba77252ea318a5672e9a0949e4495bde822c Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 15 Aug 2025 14:11:09 +0200 Subject: [PATCH 24/25] feat: improve input_data_hydro_tyndp formulation --- rules/build_electricity.smk | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 16057dff8e..621de2db25 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -536,26 +536,28 @@ rule clean_tyndp_hydro_inflows: def input_data_hydro_tyndp(w): + available_years = config_provider( + "electricity", "pemmdb_hydro_profiles", "available_years" + )(w) + planning_horizons = config_provider("scenario", "planning_horizons")(w) + safe_pyears = set( + safe_pyear( + year, + available_years, + "PEMMDB hydro", + verbose=False, + ) + for year in planning_horizons + ) + technologies = config_provider( + "electricity", "pemmdb_hydro_profiles", "technologies" + )(w) return { f"hydro_inflow_tyndp_{tech}_{pyear}": resources( f"hydro_inflows_tyndp_{tech}_{str(pyear)}.csv" ) - for pyear in set( - [ - safe_pyear( - year, - config_provider( - "electricity", "pemmdb_hydro_profiles", "available_years" - )(w), - "PEMMDB hydro", - verbose=False, - ) - for year in config_provider("scenario", "planning_horizons")(w) - ] - ) - for tech in config_provider( - "electricity", "pemmdb_hydro_profiles", "technologies" - )(w) + for pyear in safe_pyears + for tech in technologies } From 79ac928d7c6de6e74901e8011b6527f4997c726f Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 15 Aug 2025 15:47:50 +0200 Subject: [PATCH 25/25] feat: improve data cleaning --- scripts/clean_tyndp_hydro_inflows.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/scripts/clean_tyndp_hydro_inflows.py b/scripts/clean_tyndp_hydro_inflows.py index 029ec6f92b..cb364e62d1 100644 --- a/scripts/clean_tyndp_hydro_inflows.py +++ b/scripts/clean_tyndp_hydro_inflows.py @@ -68,19 +68,20 @@ def read_hydro_inflows_file( inflow_tech = ( inflow_tech.query("ShortName == 'INFLOW'") - .assign( - datetime=date_index[tech_res], - p_nom=lambda df: np.where( # calculate hourly inflow in GWh/h - df.Variable.str.contains("week"), - df[int(cyear)].div(24 * 7), # input value was either in GWh/week - df[int(cyear)].div(24), # or in GWh/day - ), - ) + .assign(datetime=date_index[tech_res]) .set_index("datetime") - .reindex(sns) # filter for snapshots only - .ffill() - .p_nom.rename(node) - .mul(1e3) # convert from GW to MW + .reindex(sns) # filter for hourly subset of snapshots only + .ffill() # upsample to hourly data + .assign( + **{ + node: lambda df: np.where( # calculate hourly inflow in MWh/h + # input value was either in GWh/week or in GWh/day + df.Variable.str.contains("week"), + df[int(cyear)] / (24 * 7 * 1e-3), + df[int(cyear)] / (24 * 1e-3), + ) + } + )[node] ) return inflow_tech