diff --git a/config/config.default.yaml b/config/config.default.yaml index f797f69f9b..ef40a25d98 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -161,6 +161,19 @@ electricity: - offwind-h2-fb-oh - offwind-h2-fl-oh + pemmdb_hydro_profiles: + enable: false + available_years: + - 2030 + - 2040 + - 2050 + technologies: + - Run of River + - Pondage + - Reservoir + - PS Open + - PS Closed + estimate_renewable_capacities: enable: true from_gem: true diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 98000d015b..ec47ea3f05 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -38,8 +38,8 @@ electricity: base_network: tyndp # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data - renewable_carriers: [onwind, solar, solar-hsat, hydro] - tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + renewable_carriers: [solar, solar-hsat, onwind] + tyndp_renewable_carriers: [hydro, onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true @@ -66,6 +66,19 @@ electricity: Wind_Onshore: - onwind + pemmdb_hydro_profiles: + enable: true + available_years: + - 2030 + - 2040 + - 2050 + technologies: + - Run of River + - Pondage + - Reservoir + - PS Open + - PS Closed + estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation technologies: diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index b72b8a3e9d..7823d8dd65 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -50,8 +50,8 @@ electricity: Link: [H2 pipeline] # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data - renewable_carriers: [onwind, solar, solar-hsat] - tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + renewable_carriers: [solar, solar-hsat, onwind] + tyndp_renewable_carriers: [hydro, onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true @@ -78,6 +78,19 @@ electricity: Wind_Onshore: - onwind + pemmdb_hydro_profiles: + enable: true + available_years: + - 2030 + - 2040 + - 2050 + technologies: + - Run of River + - Pondage + - Reservoir + - PS Open + - PS Closed + estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation technologies: diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index ea01d59ef4..edc667ce8d 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -38,6 +38,10 @@ pecd_renewable_profiles,,, -- technologies,,, -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles. +pemmdb_hydro_profiles,,, +-- enable,,bool,Activate PEMMDB hydro inflow profiles from 2024 TYNDP instead of default hydro profiles. +-- available_years,--,list,"List of years for which PEMMDB hydro inflows data is available." +-- technologies,--,list,The hydro technologies for which PEMMDB hydro inflows data is used. Technologies can be any of {Run of River, Pondage, Reservoir, PS Open, PS Closed}. estimate_renewable_capacities,,, -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. diff --git a/doc/preparation.rst b/doc/preparation.rst index fc9f5a29e9..48fcdb0fac 100644 --- a/doc/preparation.rst +++ b/doc/preparation.rst @@ -208,3 +208,25 @@ Rule ``prepare_network`` .. automodule:: prepare_network +Rule ``clean_pecd_data`` +=========================== + +.. automodule:: clean_pecd_data + +Rule ``build_renewable_profiles_pecd`` +=========================== + +.. automodule:: build_renewable_profiles_pecd + +Rule ``clean_tyndp_hydro_inflows`` +=========================== + +.. automodule:: clean_tyndp_hydro_inflows + +Rule ``build_tyndp_hydro_profile`` +=========================== + +.. automodule:: build_tyndp_hydro_profile + + + diff --git a/doc/release_notes.rst b/doc/release_notes.rst index d788b2c6a0..8c29df5cf5 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -14,6 +14,8 @@ Upcoming Open-TYNDP Release * Add complete compatibility for processing and preparation of PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP) (https://github.com/open-energy-transition/open-tyndp/pull/71). These profiles are used for the TYNDP 2024 and replace the default ERA5- and SARAH3-based profiles processed with Atlite. This implementation serves to facilitate a sub-workflow for creation of the renewable profiles, but does not yet attach them to any technologies. +* Introduce processing of PEMMDB hydro inflows data for different hydro technologies (Run of River, Pondage, Reservoir, PS Open, PS Closed) from the 2024 TYNDP to create hydro inflow profiles (https://github.com/open-energy-transition/open-tyndp/pull/77). This implementation facilitates the sub-workflow for creating the hydro inflow profiles, but it does not yet attach them to any hydro technologies. + Upcoming PyPSA-Eur Release ================ diff --git a/doc/retrieve.rst b/doc/retrieve.rst index b5eb5d65bc..94624e7d82 100644 --- a/doc/retrieve.rst +++ b/doc/retrieve.rst @@ -172,4 +172,9 @@ None. Rule ``retrieve_tyndp_pecd_data`` ==================================== -.. automodule:: retrieve_tyndp_pecd_data +.. automodule:: retrieve_additional_tyndp_data + +Rule ``retrieve_tyndp_hydro_inflows`` +==================================== + +.. automodule:: retrieve_additional_tyndp_data diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 719c93243d..621de2db25 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -513,6 +513,81 @@ rule build_hydro_profile: "../scripts/build_hydro_profile.py" +rule clean_tyndp_hydro_inflows: + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + input: + hydro_inflows_dir="data/tyndp_2024_bundle/Hydro Inflows", + busmap=resources("busmap_base_s_all.csv"), + output: + hydro_inflows_tyndp=resources( + "hydro_inflows_tyndp_{tech}_{planning_horizons}.csv" + ), + log: + logs("clean_tyndp_hydro_inflows_{tech}_{planning_horizons}.log"), + threads: 4 + benchmark: + benchmarks("clean_tyndp_hydro_inflows_{tech}_{planning_horizons}") + conda: + "../envs/environment.yaml" + script: + "../scripts/clean_tyndp_hydro_inflows.py" + + +def input_data_hydro_tyndp(w): + available_years = config_provider( + "electricity", "pemmdb_hydro_profiles", "available_years" + )(w) + planning_horizons = config_provider("scenario", "planning_horizons")(w) + safe_pyears = set( + safe_pyear( + year, + available_years, + "PEMMDB hydro", + verbose=False, + ) + for year in planning_horizons + ) + technologies = config_provider( + "electricity", "pemmdb_hydro_profiles", "technologies" + )(w) + return { + f"hydro_inflow_tyndp_{tech}_{pyear}": resources( + f"hydro_inflows_tyndp_{tech}_{str(pyear)}.csv" + ) + for pyear in safe_pyears + for tech in technologies + } + + +rule build_tyndp_hydro_profile: + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + planning_horizons=config_provider("scenario", "planning_horizons"), + available_years=config_provider( + "electricity", "pemmdb_hydro_profiles", "available_years" + ), + technologies=config_provider( + "electricity", "pemmdb_hydro_profiles", "technologies" + ), + input: + unpack(input_data_hydro_tyndp), + output: + profile=resources("profile_pemmdb_hydro.nc"), + log: + logs("build_tyndp_hydro_profile.log"), + benchmark: + benchmarks("build_tyndp_hydro_profile") + resources: + mem_mb=5000, + conda: + "../envs/environment.yaml" + script: + "../scripts/build_tyndp_hydro_profile.py" + + rule build_line_rating: params: snapshots=config_provider("snapshots"), diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 8c1eabded0..9153d567cd 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1573,6 +1573,11 @@ rule prepare_sector_network: if config_provider("sector", "h2_topology_tyndp")(w) else [] ), + profile_pemmdb_hydro=branch( + config_provider("electricity", "pemmdb_hydro_profiles", "enable"), + resources("profile_pemmdb_hydro.nc"), + [], + ), output: resources( "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc" diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 11ed5d16a7..06670100cf 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -199,15 +199,28 @@ if config["enable"]["retrieve"]: params: # TODO Integrate into Zenodo tyndp data bundle tyndp_bundle="data/tyndp_2024_bundle", + url="https://storage.googleapis.com/open-tyndp-data-store/PECD.zip", + source="PECD", output: dir=directory("data/tyndp_2024_bundle/PECD"), log: "logs/retrieve_tyndp_pecd_data.log", retries: 2 script: - "../scripts/retrieve_tyndp_pecd_data.py" + "../scripts/retrieve_additional_tyndp_data.py" + + use rule retrieve_tyndp_pecd_data as retrieve_tyndp_hydro_inflows with: + params: + # TODO Integrate into Zenodo tyndp data bundle + url="https://storage.googleapis.com/open-tyndp-data-store/Hydro_Inflows.zip", + source="hydro inflows", + output: + dir=directory("data/tyndp_2024_bundle/Hydro Inflows"), + log: + "logs/retrieve_tyndp_hydro_inflows.log", ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data + ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_hydro_inflows > clean_tyndp_hydro_inflows rule retrieve_countries_centroids: output: diff --git a/scripts/build_tyndp_hydro_profile.py b/scripts/build_tyndp_hydro_profile.py new file mode 100644 index 0000000000..0d85f83f5d --- /dev/null +++ b/scripts/build_tyndp_hydro_profile.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Contributors to Open-TYNDP +# +# SPDX-License-Identifier: MIT +""" +Builds hydroelectric inflow time-series for each country based on PEMMDB v2.4 hydro inflow data from the 2024 TYNDP. + +Outputs +------- + +- ``resources/profile_pemmdb_hydro.nc``: + + =================== ================ ========================================================= + Field Dimensions Description + =================== ================ ========================================================= + inflow bus, time, Inflow to the state of charge (in MW), + year, hydro_tech e.g. due to river inflow in hydro reservoir. + =================== ================ ========================================================= +""" + +import logging + +import pandas as pd +import xarray as xr +from tqdm.contrib.itertools import product + +from scripts._helpers import ( + configure_logging, + get_snapshots, + safe_pyear, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake("build_tyndp_hydro_profile") + configure_logging(snakemake) + set_scenario_config(snakemake) + + time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + + years_in_time = pd.DatetimeIndex(time).year.unique() + + technologies = snakemake.params.technologies + pyears = snakemake.params.planning_horizons + + inflows = [] + + for year, technology in product(pyears, technologies): + logger.info(f"Extracting hydro inflows for {technology} in {year}") + year_i = year + # falling back to latest available pyear if not in list of available years + year = safe_pyear( + int(year), + available_years=snakemake.params.available_years, + source="PEMMDB hydro inflow", + ) + + inflow = ( + pd.read_csv( + snakemake.input[f"hydro_inflow_tyndp_{technology}_{year}"], + parse_dates=True, + index_col=0, + ) + .rename_axis("time") + .reset_index() + .melt(id_vars=["time"], var_name="bus", value_name="profile") + .assign(year=year_i, hydro_tech=technology) + .set_index(["bus", "time", "year", "hydro_tech"]) + .to_xarray() + ) + + inflows.append(inflow) + + ds = xr.merge(inflows) + ds = ds.sel(time=time) + + ds.to_netcdf(snakemake.output.profile) diff --git a/scripts/clean_tyndp_hydro_inflows.py b/scripts/clean_tyndp_hydro_inflows.py new file mode 100644 index 0000000000..cb364e62d1 --- /dev/null +++ b/scripts/clean_tyndp_hydro_inflows.py @@ -0,0 +1,163 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: MIT +""" +Loads and cleans the available hydro inflow data from TYNDP data bundle for a given + +* climate year, +* planning horizon, +* hydro technology. + +Input data for TYNDP 2024 comes from PEMMDB v2.4. + +Outputs +------- +Cleaned csv file with hourly hydro inflow time series in MW per region. +""" + +import logging +import multiprocessing as mp +import os +from functools import partial +from pathlib import Path + +import numpy as np +import pandas as pd +from tqdm import tqdm + +from scripts._helpers import ( + configure_logging, + get_snapshots, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + + +def read_hydro_inflows_file( + node: str, + hydro_inflows_dir: str, + cyear: str, + pyear: str, + hydro_tech: str, + sns: pd.DatetimeIndex, + date_index: dict, +) -> pd.Series: + fn = Path( + hydro_inflows_dir, + pyear, + f"PEMMDB_{node.replace('GB', 'UK')}_Hydro_Inflows_{pyear}.xlsx", + ) + + if not os.path.isfile(fn): + return None + + inflow_tech = pd.read_excel( + fn, + skiprows=1, + usecols=lambda name: name == "Day" + or name == "Week" + or name == "ShortName" + or name == "Variable" + or name == int(cyear), + sheet_name=f"{hydro_tech} - Year Dependent", + ) + + # infer resolution of data for each technology + tech_res = "w" if "Week" in inflow_tech.columns else "d" + + inflow_tech = ( + inflow_tech.query("ShortName == 'INFLOW'") + .assign(datetime=date_index[tech_res]) + .set_index("datetime") + .reindex(sns) # filter for hourly subset of snapshots only + .ffill() # upsample to hourly data + .assign( + **{ + node: lambda df: np.where( # calculate hourly inflow in MWh/h + # input value was either in GWh/week or in GWh/day + df.Variable.str.contains("week"), + df[int(cyear)] / (24 * 7 * 1e-3), + df[int(cyear)] / (24 * 1e-3), + ) + } + )[node] + ) + + return inflow_tech + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake( + "clean_tyndp_hydro_inflows", + clusters="all", + planning_horizons=2030, + tech="Run of River", + ) + configure_logging(snakemake) + set_scenario_config(snakemake) + + # Climate year from snapshots + sns = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + cyear = sns[0].year + date_index = { + "w": pd.date_range( + start=f"{cyear}-01-01", + periods=53, # 53 weeks + freq="7D", + ), + "d": pd.date_range( + start=f"{cyear}-01-01", + periods=366, # 366 days (incl. first day of next year) + freq="D", + ), + } + if int(cyear) < 1982 or int(cyear) > 2019: + logger.warning( + f"Snapshot year {cyear} doesn't match available TYNDP data. Falling back to 2009." + ) + cyear = 2009 + + # Planning year + pyear = str(snakemake.wildcards.planning_horizons) + + # Parameters + onshore_buses = pd.read_csv(snakemake.input.busmap, index_col=0) + nodes = onshore_buses.index + hydro_inflows_dir = snakemake.input.hydro_inflows_dir + hydro_tech = str(snakemake.wildcards.tech) + + # Load and prep inflow data + tqdm_kwargs = { + "ascii": False, + "unit": " nodes", + "total": len(nodes), + "desc": "Loading TYNDP hydro inflows data", + } + + func = partial( + read_hydro_inflows_file, + hydro_inflows_dir=hydro_inflows_dir, + cyear=cyear, + pyear=pyear, + hydro_tech=hydro_tech, + sns=sns, + date_index=date_index, + ) + + with mp.Pool(processes=snakemake.threads) as pool: + inflows = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + + inflows_df = ( + pd.concat(inflows, axis=1) + .reindex( + nodes, + axis=1, + ) # include missing node data with empty columns + .fillna(0.0) # fill missing data with zero values + ) + + inflows_df.to_csv(snakemake.output.hydro_inflows_tyndp) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index bfa1a466ec..b4491e7ce2 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -7523,7 +7523,7 @@ def add_import_options( profiles = { key: snakemake.input[key] for key in snakemake.input.keys() - if key.startswith("profile") + if key.startswith("profile") and "hydro" not in key } pecd_renewable_profiles_techs = snakemake.params.electricity[ "pecd_renewable_profiles" diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_additional_tyndp_data.py similarity index 56% rename from scripts/retrieve_tyndp_pecd_data.py rename to scripts/retrieve_additional_tyndp_data.py index 8d1666da7a..e9ccc49db6 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_additional_tyndp_data.py @@ -2,16 +2,24 @@ # # SPDX-License-Identifier: MIT """ -The TYNDP PECD data contains input data for the 2024 TYNDP scenario building process. - -This rule downloads the TYNDP PECD v3.1 data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` +Retrieves additional TYNDP data not included in the Zenodo TYNDP data bundle. +Downloads the zip file from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` subdirectory, such that all files of the TYNDP bundle are stored in it. The original data is published by ENTSO-E and ENTSOG under Creative Commons Attribution 4.0 International License (CC-BY 4.0) and can be found under https://2024.entsos-tyndp-scenarios.eu/download/. +Currently, this is used for two additional datasets: +* TYNDP PECD data: The TYNDP PECD v3.1 data contains input data for the 2024 TYNDP scenario building process. +* TYNDP hydro inflows: The TYNDP hydro inflow data from PEMMDB v2.4 contains hydro inflow data for different hydro technologies: + * Run of River, + * Pondage, + * Reservoir, + * PS Open, + * PS Closed + **Outputs** -- ``data/tyndp_2024_bundle/PECD``: PECD input data for TYNDP 2024 scenario building +- ``data/tyndp_2024_bundle/``: Additional input dataset for TYNDP 2024 scenario building """ @@ -24,9 +32,7 @@ logger = logging.getLogger(__name__) -# Define the base URL -# TODO: retrieve_tyndp_pecd_data needs to be deprecated once PECD data is added to the TYNDP data bundle -url = "https://storage.googleapis.com/open-tyndp-data-store/PECD.zip" +# TODO: retrieve_additional_tyndp_data needs to be deprecated once all TYNDP data is added to the TYNDP data bundle if __name__ == "__main__": if "snakemake" not in globals(): @@ -41,20 +47,24 @@ set_scenario_config(snakemake) disable_progress = snakemake.config["run"].get("disable_progressbar", False) + source = snakemake.params.source to_fn = snakemake.output.dir tyndp_bundle_fn = Path(rootpath, snakemake.params["tyndp_bundle"]) to_fn_zp = to_fn + ".zip" + # define url + url = snakemake.params.url + # download .zip file - logger.info(f"Downloading TYNDP PECD data from '{url}'.") + logger.info(f"Downloading TYNDP {source} data from '{url}'.") progress_retrieve(url, to_fn_zp, disable=disable_progress) # extract - logger.info("Extracting TYNDP PECD data.") + logger.info(f"Extracting TYNDP {source} data.") with zipfile.ZipFile(to_fn_zp, "r") as zip_ref: zip_ref.extractall(tyndp_bundle_fn) # remove .zip file os.remove(to_fn_zp) - logger.info(f"TYNDP PECD data available in '{to_fn}'.") + logger.info(f"TYNDP {source} data available in '{to_fn}'.")