diff --git a/config/config.default.yaml b/config/config.default.yaml index 5f046bbc23..393a129bb1 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -144,6 +144,11 @@ electricity: pecd_renewable_profiles: enable: false + fill_gaps_method: zero + available_years: + - 2030 + - 2040 + - 2050 technologies: Wind_Offshore: - offwind-ac-fb-r @@ -392,6 +397,10 @@ load: enable: true interpolate_limit: 3 time_shift_for_large_gaps: 1w + available_years_tyndp: + - 2030 + - 2040 + - 2050 manual_adjustments: true scaling_factor: 1.0 fixed_year: false diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 01285c3b86..98000d015b 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -37,11 +37,18 @@ co2_budget: electricity: base_network: tyndp - renewable_carriers: [solar, solar-hsat, onwind, hydro] - tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data + renewable_carriers: [onwind, solar, solar-hsat, hydro] + tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true + fill_gaps_method: zero + # Complete PECD data is only available for the years 2030, 2040 + # TODO: adjust once udpated 2050 PECD data is available + available_years: + - 2030 + - 2040 technologies: Wind_Offshore: - offwind-ac-fb-r @@ -52,6 +59,12 @@ electricity: - offwind-dc-fl-oh - offwind-h2-fb-oh - offwind-h2-fl-oh + LFSolarPVRooftop: + - solar-pv-rooftop + LFSolarPVUtility: + - solar-pv-utility + Wind_Onshore: + - onwind estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation @@ -76,6 +89,10 @@ load: source: tyndp # opsd, tyndp fill_gaps: enable: false + available_years_tyndp: + - 2030 + - 2040 + - 2050 manual_adjustments: false supplement_synthetic: false diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index bfc66b1777..d23af674c0 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -48,11 +48,18 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar, solar-hsat, onwind] - tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data + renewable_carriers: [onwind, solar, solar-hsat] + tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true + fill_gaps_method: zero + # Complete PECD data is only available for the years 2030, 2040 + # TODO: adjust once udpated 2050 PECD data is available + available_years: + - 2030 + - 2040 technologies: Wind_Offshore: - offwind-ac-fb-r @@ -63,6 +70,12 @@ electricity: - offwind-dc-fl-oh - offwind-h2-fb-oh - offwind-h2-fl-oh + LFSolarPVRooftop: + - solar-pv-rooftop + LFSolarPVUtility: + - solar-pv-utility + Wind_Onshore: + - onwind estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation @@ -102,6 +115,10 @@ load: source: tyndp # opsd, tyndp fill_gaps: enable: false + available_years_tyndp: + - 2030 + - 2040 + - 2050 manual_adjustments: false supplement_synthetic: false diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 2e8a61e495..6ce61b39a7 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -32,7 +32,9 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv-rooftop, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh}",List of TYNDP renewable generators to include in the model. Technologies covered by specified TYNDP renewable carriers need to be removed from `estimate_renewable_carriers` technology list. pecd_renewable_profiles,,, --- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {LFSolarPVUtility, LFSolarPVRooftop, Wind_Offshore, Wind_Onshore, CSP_noStorage, CSP_withStorage_7h_dispatched, CSP_withStorage_7h_preDispatch}. +-- fill_gaps_method,,str,The chosen method for filling gaps in the PECD data to the modelled nodes. Can be either `zero` to fill with zero values or any other aggregation method such as `mean`, `median`, `max` or similar. +-- available_years,--,list,"List of years for which PECD data is available." -- technologies,,, -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles. diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index 79e75f2029..68e5ce4dbf 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -4,6 +4,7 @@ fill_gaps,--,--,"Gaps filling strategy used." -- enable,bool,"{true, false}","Whether to fill gaps using interpolation for small gaps and time shift for large gaps." -- interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly." -- time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings." +available_years_tyndp,--,list,"List of years for which TYNDP demand data is available." manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." scaling_factor,--,float,"Global correction factor for the load time series." fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year" diff --git a/doc/data-retrieval.rst b/doc/data-retrieval.rst index 5ed4bb51a1..c6af8dd9c6 100644 --- a/doc/data-retrieval.rst +++ b/doc/data-retrieval.rst @@ -243,3 +243,11 @@ Data in this section is retrieved and extracted in rules specified in ``rules/re - **Link:** https://cdn.jsdelivr.net/gh/gavinr/world-countries-centroids@v1.0.0/dist/countries.geojson - **License:** MIT License - **Description:** Contains centroids of world countries in GeoJson. + +``data/tyndp_2024_bundle/PECD`` + +- **Source:** TYNDP 2024 Scenarios, https://2024.entsos-tyndp-scenarios.eu/download/ +- **Link:** https://storage.googleapis.com/open-tyndp-data-store/PECD.zip +- **License:** CC-BY 4.0 +- **Description:** Contains hourly PECD v3.1 capacity factor time series for Solar PV rooftop, + Solar PV utility, Onshore Wind, Offshore Wind and Solar CSP. diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 68a3873d2c..e4f37d392b 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -11,7 +11,7 @@ Release Notes Upcoming Open-TYNDP Release ================ - +* Add complete compatibility for processing and preparation of PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP) (https://github.com/open-energy-transition/open-tyndp/pull/71). These profiles are used for the TYNDP 2024 and replace the default ERA5- and SARAH3-based profiles processed with Atlite. This implementation serves to facilitate a sub-workflow for creation of the renewable profiles, but does not yet attach them to any technologies. Upcoming PyPSA-Eur Release diff --git a/doc/retrieve.rst b/doc/retrieve.rst index 40f4fa7469..378d43727b 100644 --- a/doc/retrieve.rst +++ b/doc/retrieve.rst @@ -167,3 +167,8 @@ None. **Outputs** - ``data/countries_centroids.geojson`` + +Rule ``retrieve_tyndp_pecd_data`` +==================================== + +.. automodule:: retrieve_tyndp_pecd_data diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index c49423c539..7f5d7b3a36 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -2,6 +2,8 @@ # # SPDX-License-Identifier: MIT +from scripts._helpers import safe_pyear + def input_elec_demand(w): return { @@ -397,6 +399,12 @@ rule clean_pecd_data: params: snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), + fill_gaps_method=config_provider( + "electricity", "pecd_renewable_profiles", "fill_gaps_method" + ), + available_years=config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + ), input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), @@ -417,19 +425,26 @@ rule clean_pecd_data: def input_data_pecd(w): + available_years = config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + )(w) + planning_horizons = config_provider("scenario", "planning_horizons")(w) + safe_pyears = set( + safe_pyear(year, available_years, "PECD", verbose=False) + for year in planning_horizons + ) return { f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv") - for pyear in set( - config_provider("scenario", "planning_horizons")(w) - ).intersection([2030, 2040]) - # Complete PECD data is only available for the years 2030, 2040 - # TODO: adjust if udpated 2050 data available + for pyear in safe_pyears } rule build_renewable_profiles_pecd: params: planning_horizons=config_provider("scenario", "planning_horizons"), + available_years=config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + ), input: unpack(input_data_pecd), output: @@ -1031,6 +1046,7 @@ if config["load"]["source"] == "tyndp": planning_horizons=config_provider("scenario", "planning_horizons"), snapshots=config_provider("snapshots"), scenario=config_provider("tyndp_scenario"), + available_years=config_provider("load", "available_years_tyndp"), input: electricity_demand=directory("data/tyndp_2024_bundle/Demand Profiles"), output: diff --git a/rules/collect.smk b/rules/collect.smk index f0f84df3fc..9783b031a1 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -90,3 +90,27 @@ rule plot_power_networks_clustered: **config["scenario"], run=config["run"]["name"], ), + + +rule clean_pecd_datas: + input: + lambda w: expand( + resources("pecd_data_{technology}_{planning_horizons}.csv"), + **config["scenario"], + run=config["run"]["name"], + technology=config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w), + ), + + +rule build_renewable_profiles_pecds: + input: + lambda w: expand( + resources("profile_pecd_{clusters}_{technology}.nc"), + **config["scenario"], + run=config["run"]["name"], + technology=config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w), + ), diff --git a/scripts/_helpers.py b/scripts/_helpers.py index f1538ccca4..ff4e3a7f92 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -9,6 +9,7 @@ import os import re import time +from bisect import bisect_right from functools import partial, wraps from pathlib import Path from tempfile import NamedTemporaryFile @@ -1129,3 +1130,49 @@ def extract_grid_data_tyndp( links.index = links.apply(make_index, axis=1, prefix=carrier) return links + + +def safe_pyear( + year: int | str, + available_years: list = [2030, 2040, 2050], + source: str = "TYNDP", + verbose: bool = True, +): + """ + Checks and adjusts whether a given pyear is in the available years of a given data source. If not, it falls back to the previous available year. + + Parameters + ---------- + year : int + Planning horizon year which will be checked and possibly adjusted to previous available year. + available_years : list, optional + List of available years. Defaults to [2030, 2040, 2050]. + source : str, optional + Source of the data for which availability will be checked. For logging purpose only. Defaults to "TYNDP". + verbose : bool, optional + Whether to activate verbose logging. Defaults to True. + + Returns + ------- + year_new : int + Safe pyear adjusted for available years + """ + + if not available_years: + raise ValueError( + "No `available_years` provided. Expected a non-empty list of years." + ) + if not isinstance(year, int): + year = int(year) + if year not in available_years: + year_new = available_years[ + bisect_right(sorted(available_years), year, lo=1) - 1 + ] + if verbose: + logger.warning( + f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}." + ) + else: + year_new = year + + return year_new diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 5c08ba485e..68c2a55960 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -102,6 +102,12 @@ def add_existing_renewables( irena = irena.unstack().reset_index() + if not set(tech_map).intersection(renewable_carriers): + # TODO: adjust once TYNDP existing capacities are added. + logger.info( + "No existing capacities to add for specified renewable carriers. Existing capacities of TYNDP renewable carriers will be added separately." + ) + for carrier, tech in tech_map.items(): if carrier not in renewable_carriers: continue @@ -145,7 +151,9 @@ def add_existing_renewables( df_agg.at[name, "bus"] = bus df_agg.at[name, "resource_class"] = bin_id - df_agg["resource_class"] = df_agg["resource_class"].fillna(0) + df_agg["resource_class"] = ( + df_agg["resource_class"].fillna(0) if "resource_class" in df_agg.columns else 0 + ) def add_power_capacities_installed_before_baseyear( diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 67646349b6..0fbcc8c410 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -22,12 +22,12 @@ import logging -import numpy as np import pandas as pd import xarray as xr from scripts._helpers import ( configure_logging, + safe_pyear, set_scenario_config, ) @@ -55,16 +55,10 @@ f"Extract PECD capacity factor time series for year {year} for technology {technology}..." ) year_i = year - if int(year) not in [2030, 2040, 2050]: - year = np.clip(10 * (year // 10), 2030, 2050) - logger.warning( - f"TYNDP PECD data unavailable for planning horizon. Falling back to previous available year {year}." - ) - if year == 2050: - logger.warning( - "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data." - ) - year = 2040 + # falling back to latest available pyear if not in list of available years + year = safe_pyear( + year, available_years=snakemake.params.available_years, source="PECD" + ) profile = ( pd.read_csv( diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index c582b08b1d..2f8f6567ee 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -3,15 +3,15 @@ # SPDX-License-Identifier: MIT """ Loads and cleans the available PECD capacity factor generation time series based on PECD weather data. -The script is executed for a given technology, and planning horizon. Technologies can be one of: +The script is executed for a given technology and planning horizon. Technologies can be one of: - * CSP_noStorage, - * CSP_withStorage, - * LFSolarPV, - * LFSolarPVRooftop, * LFSolarPVUtility, + * LFSolarPVRooftop, * Wind_Offshore, - * Wind_Onshore. + * Wind_Onshore, + * CSP_noStorage, + * CSP_withStorage_7h_dispatched, + * CSP_withStorage_7h_preDispatch (note: includes cf > 1 for when thermal storage can be used). Outputs ------- @@ -30,6 +30,7 @@ from scripts._helpers import ( configure_logging, get_snapshots, + safe_pyear, set_scenario_config, ) @@ -40,18 +41,32 @@ def read_pecd_file( node: str, dir_pecd: str, cyear: str, - pyear: str, + pyear: int, technology: str, sns: pd.DatetimeIndex, ): - fn = Path(dir_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + fn = Path( + dir_pecd, + str(pyear), + f"PECD_{technology}_{pyear}_{node.replace('GB', 'UK')}_edition 2023.2.csv", + ) + # PECD only differentiates between utility and rooftop PV for some nodes + if not os.path.isfile(fn) and "LFSolarPV" in technology: + fn = Path(str(fn).replace(technology, "LFSolarPV")) if not os.path.isfile(fn): + logger.warning(f"Missing data for {technology} in {node} in {pyear}.") return None + # Malta CSP data file has an extra header row that must be skipped + if node == "MT00" and technology == "CSP_noStorage" and pyear == 2040: + skiprows = 11 + else: + skiprows = 10 + pecd_bus = pd.read_csv( fn, - skiprows=10, # first ten rows contain only file metadata + skiprows=skiprows, # first rows contain only file metadata usecols=lambda name: name == "Date" or name == "Hour" or name == str(cyear) @@ -94,10 +109,13 @@ def read_pecd_file( ) cyear = 2009 - # Planning year - pyear = str(snakemake.wildcards.planning_horizons) + # Planning year (falls back to latest available pyear if not in list of available years) + pyear = safe_pyear( + snakemake.wildcards.planning_horizons, + available_years=snakemake.params.available_years, + source="PECD", + ) - # TODO: find solution for solar profiles being differentiated between Utility and Rooftop for Italy # Technology as in PECD terminology pecd_tech = snakemake.wildcards.technology @@ -105,11 +123,15 @@ def read_pecd_file( onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) nodes = ( - offshore_buses.index if pecd_tech == "Wind_Offshore" else onshore_buses.index + offshore_buses.index.str.replace( + "UK", "GB", regex=True + ) # replace UK with GB for naming convention + if pecd_tech == "Wind_Offshore" + else onshore_buses.index ) dir_pecd = snakemake.input.dir_pecd - # Load and prep electricity demand + # Load and prep pecd data tqdm_kwargs = { "ascii": False, "unit": " nodes", @@ -127,16 +149,24 @@ def read_pecd_file( ) with mp.Pool(processes=snakemake.threads) as pool: - demand = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + pecd = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + if all(data is None for data in pecd): + raise ValueError( + f"No PECD data found for {pecd_tech} in {pyear}. Please specify a technology covered within the TYNDP PECD data." + ) + pecd_df = pd.concat(pecd, axis=1) + fill_na = ( + pd.Series(0.0, index=pecd_df.index) + if snakemake.params.fill_gaps_method == "zero" + else pecd_df.agg(snakemake.params.fill_gaps_method, axis=1) + ) pecd_df = ( - pd.concat(demand, axis=1) - .reindex( - nodes, axis=1, fill_value=0.0 - ) # include missing node data with empty columns - .rename( - columns=lambda x: x.replace("UK", "GB") - ) # replace UK with GB for naming convention + pecd_df.reindex( + nodes, axis=1 + ).where( # include missing node data with empty columns + lambda df: df.notna(), fill_na, axis=0 + ) # fill missing node data with configured aggregation method ) pecd_df.to_csv(snakemake.output.pecd_data_clean) diff --git a/scripts/clean_tyndp_demand.py b/scripts/clean_tyndp_demand.py index 3a03c391f5..24ee5479b8 100644 --- a/scripts/clean_tyndp_demand.py +++ b/scripts/clean_tyndp_demand.py @@ -12,15 +12,21 @@ from functools import partial from pathlib import Path -import numpy as np import pandas as pd -from _helpers import configure_logging, get_snapshots, set_scenario_config +from _helpers import ( + configure_logging, + get_snapshots, + safe_pyear, + set_scenario_config, +) from tqdm import tqdm logger = logging.getLogger(__name__) -def load_elec_demand(fn: str, scenario: str, pyear: int, cyear: int): +def load_elec_demand( + fn: str, scenario: str, pyear: int, cyear: int, available_years: list +): """ Load electricity demand files into dictionary of dataframes. Filter for specific climatic year and format data. """ @@ -28,12 +34,8 @@ def load_elec_demand(fn: str, scenario: str, pyear: int, cyear: int): # handle intermediate years # TODO: Possibly improve this with linear interpolation for 2035 and 2045 - if pyear not in [2030, 2040, 2050]: - pyear = np.clip(10 * (pyear // 10), 2030, 2050) - logger.warning( - "Planning horizon doesn't match available 2024 TYNDP electricity demand data. " - f"Falling back to previous available year {pyear}." - ) + pyear = safe_pyear(pyear, available_years=available_years, source="TYNDP demand") + if scenario == "NT": if pyear == 2050: logger.warning( @@ -132,6 +134,7 @@ def load_elec_demand(fn: str, scenario: str, pyear: int, cyear: int): snakemake.input.electricity_demand, scenario, cyear=cyear, + available_years=snakemake.params.available_years, ) with mp.Pool(processes=snakemake.threads) as pool: diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 8dc9ffdb1b..8d1666da7a 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -6,6 +6,8 @@ This rule downloads the TYNDP PECD v3.1 data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` subdirectory, such that all files of the TYNDP bundle are stored in it. +The original data is published by ENTSO-E and ENTSOG under Creative Commons Attribution 4.0 International License (CC-BY 4.0) +and can be found under https://2024.entsos-tyndp-scenarios.eu/download/. **Outputs**