From 4b8f2a23dc8671245d7bfe02af5023964bb46efe Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 11 Jul 2025 14:19:18 +0200 Subject: [PATCH 01/29] feat: add configuration for pecd fill_gaps_method for missing data --- config/config.default.yaml | 1 + config/config.tyndp.yaml | 1 + config/test/config.tyndp.yaml | 1 + doc/configtables/electricity.csv | 1 + rules/build_electricity.smk | 3 +++ scripts/clean_pecd_data.py | 18 ++++++++++++------ 6 files changed, 19 insertions(+), 6 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 79667abe83..7c7814c768 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -116,6 +116,7 @@ electricity: pecd_renewable_profiles: enable: false + fill_gaps_method: zero technologies: Wind_Offshore: - offwind-ac-fb-r diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 45c07636fd..d59f5a363f 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -45,6 +45,7 @@ electricity: pecd_renewable_profiles: enable: true + fill_gaps_method: zero technologies: Wind_Offshore: - offwind-ac-fb-r diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 9e781ec6e5..c3d66f988d 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -49,6 +49,7 @@ electricity: pecd_renewable_profiles: enable: true + fill_gaps_method: zero technologies: Wind_Offshore: - offwind-ac-fb-r diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 2e8a61e495..2999b71439 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -33,6 +33,7 @@ renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, off tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv-rooftop, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh}",List of TYNDP renewable generators to include in the model. Technologies covered by specified TYNDP renewable carriers need to be removed from `estimate_renewable_carriers` technology list. pecd_renewable_profiles,,, -- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. +-- fill_gaps_method,,str,The chosen method for filling gaps in the PECD data to the modelled nodes. Can be either `zero` to fill with zero values or any other aggregation method such as `mean`, `median`, `max` or similar. -- technologies,,, -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 0797d3aef8..084ade8f5c 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -397,6 +397,9 @@ rule clean_pecd_data: params: snapshots=config_provider("snapshots"), drop_leap_day=config_provider("enable", "drop_leap_day"), + fill_gaps_method=config_provider( + "electricity", "pecd_renewable_profiles", "fill_gaps_method" + ), input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index c582b08b1d..c60f1a5de0 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -109,7 +109,7 @@ def read_pecd_file( ) dir_pecd = snakemake.input.dir_pecd - # Load and prep electricity demand + # Load and prep pecd data tqdm_kwargs = { "ascii": False, "unit": " nodes", @@ -127,16 +127,22 @@ def read_pecd_file( ) with mp.Pool(processes=snakemake.threads) as pool: - demand = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + pecd = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + pecd_df = pd.concat(pecd, axis=1) + fill_na = ( + pd.Series(0.0, index=pecd_df.index) + if snakemake.params.fill_gaps_method == "zero" + else pecd_df.agg(snakemake.params.fill_gaps_method, axis=1) + ) pecd_df = ( - pd.concat(demand, axis=1) - .reindex( - nodes, axis=1, fill_value=0.0 - ) # include missing node data with empty columns + pecd_df.reindex(nodes, axis=1) # include missing node data with empty columns .rename( columns=lambda x: x.replace("UK", "GB") ) # replace UK with GB for naming convention + .where( + lambda df: df.notna(), fill_na, axis=0 + ) # fill missing node data with configured aggregation method ) pecd_df.to_csv(snakemake.output.pecd_data_clean) From 1224c36d7e50adf3e3c65ca42b4091c85eb45d61 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 14 Jul 2025 15:54:07 +0200 Subject: [PATCH 02/29] feat: add SolarPV for utility and rooftop with differentiation --- config/config.tyndp.yaml | 6 +++++- config/test/config.tyndp.yaml | 8 ++++++-- doc/configtables/electricity.csv | 2 +- scripts/clean_pecd_data.py | 14 ++++++++------ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index d59f5a363f..127e8972b1 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -41,7 +41,7 @@ electricity: base_network: tyndp-raw renewable_carriers: [solar, solar-hsat, onwind, hydro] - tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + tyndp_renewable_carriers: [solar-pv, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true @@ -56,6 +56,10 @@ electricity: - offwind-dc-fl-oh - offwind-h2-fb-oh - offwind-h2-fl-oh + LFSolarPVRooftop: + - solar-pv-rooftop + LFSolarPVUtility: + - solar-pv-utility estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index c3d66f988d..dc8ac288d8 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -44,8 +44,8 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar, solar-hsat, onwind] - tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + renewable_carriers: [solar-hsat, onwind] + tyndp_renewable_carriers: [solar-pv, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true @@ -60,6 +60,10 @@ electricity: - offwind-dc-fl-oh - offwind-h2-fb-oh - offwind-h2-fl-oh + LFSolarPVRooftop: + - solar-pv-rooftop + LFSolarPVUtility: + - solar-pv-utility estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 2999b71439..29ef77ed30 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -32,7 +32,7 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv-rooftop, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh}",List of TYNDP renewable generators to include in the model. Technologies covered by specified TYNDP renewable carriers need to be removed from `estimate_renewable_carriers` technology list. pecd_renewable_profiles,,, --- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPVUtility, LFSolarPVRooftop, Wind_Offshore, Wind_Onshore}. -- fill_gaps_method,,str,The chosen method for filling gaps in the PECD data to the modelled nodes. Can be either `zero` to fill with zero values or any other aggregation method such as `mean`, `median`, `max` or similar. -- technologies,,, -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index c60f1a5de0..93da8d4b31 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -5,11 +5,9 @@ Loads and cleans the available PECD capacity factor generation time series based on PECD weather data. The script is executed for a given technology, and planning horizon. Technologies can be one of: - * CSP_noStorage, - * CSP_withStorage, - * LFSolarPV, - * LFSolarPVRooftop, + * CSP * LFSolarPVUtility, + * LFSolarPVRooftop, * Wind_Offshore, * Wind_Onshore. @@ -44,6 +42,9 @@ def read_pecd_file( technology: str, sns: pd.DatetimeIndex, ): + # PECD only differentiates IT nodes between utility and rooftop PV + if "LFSolarPV" in technology and "IT" not in node: + technology = "LFSolarPV" fn = Path(dir_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") if not os.path.isfile(fn): @@ -97,7 +98,6 @@ def read_pecd_file( # Planning year pyear = str(snakemake.wildcards.planning_horizons) - # TODO: find solution for solar profiles being differentiated between Utility and Rooftop for Italy # Technology as in PECD terminology pecd_tech = snakemake.wildcards.technology @@ -105,7 +105,9 @@ def read_pecd_file( onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) nodes = ( - offshore_buses.index if pecd_tech == "Wind_Offshore" else onshore_buses.index + offshore_buses.index + if pecd_tech == "Wind_Offshore" + else onshore_buses.index.str.replace("GB", "UK", regex=True) ) dir_pecd = snakemake.input.dir_pecd From d5015972926d61f492bee7cc5460abf3bcedb8d9 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 15 Jul 2025 09:40:45 +0200 Subject: [PATCH 03/29] fix: adjust config name of solar-pv carrier --- config/config.tyndp.yaml | 2 +- config/test/config.tyndp.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 127e8972b1..5eaf08de9f 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -41,7 +41,7 @@ electricity: base_network: tyndp-raw renewable_carriers: [solar, solar-hsat, onwind, hydro] - tyndp_renewable_carriers: [solar-pv, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + tyndp_renewable_carriers: [solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index dc8ac288d8..1e78725b91 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -45,7 +45,7 @@ electricity: Link: [H2 pipeline] renewable_carriers: [solar-hsat, onwind] - tyndp_renewable_carriers: [solar-pv, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + tyndp_renewable_carriers: [solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true From 89a89ad6faa1f53659fb47c00d43d10738b0016b Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 15 Jul 2025 17:09:52 +0200 Subject: [PATCH 04/29] fix: make LFSolarPV import more generic to differentiation of more nodes --- scripts/clean_pecd_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 93da8d4b31..3f98dd595e 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -42,11 +42,11 @@ def read_pecd_file( technology: str, sns: pd.DatetimeIndex, ): - # PECD only differentiates IT nodes between utility and rooftop PV - if "LFSolarPV" in technology and "IT" not in node: - technology = "LFSolarPV" fn = Path(dir_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + # PECD only differentiates between utility and rooftop PV for some nodes + if not os.path.isfile(fn) and "LFSolarPV" in technology: + fn = Path(str(fn).replace(technology, "LFSolarPV")) if not os.path.isfile(fn): return None From 148b5081a03ebfdac9a30d2824062028dad66809 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 15 Jul 2025 17:14:33 +0200 Subject: [PATCH 05/29] feat: add Exception if no input data for specified PECD tech --- scripts/clean_pecd_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 3f98dd595e..f839be9ca4 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -131,6 +131,10 @@ def read_pecd_file( with mp.Pool(processes=snakemake.threads) as pool: pecd = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + if all(data is None for data in pecd): + raise ValueError( + f"No PECD data found for {pecd_tech}. Please specify a technology covered within the TYNDP PECD data." + ) pecd_df = pd.concat(pecd, axis=1) fill_na = ( pd.Series(0.0, index=pecd_df.index) From 265b6ac1ec1f10f3e9370bb5a553c53ebd2b8615 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Tue, 15 Jul 2025 17:58:25 +0200 Subject: [PATCH 06/29] feat: add onshore wind to tyndp renewable carriers and pecd profiles --- config/config.tyndp.yaml | 6 ++++-- config/test/config.tyndp.yaml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 5eaf08de9f..252b29a201 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -40,8 +40,8 @@ co2_budget: electricity: base_network: tyndp-raw - renewable_carriers: [solar, solar-hsat, onwind, hydro] - tyndp_renewable_carriers: [solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + renewable_carriers: [solar-hsat, hydro] + tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true @@ -60,6 +60,8 @@ electricity: - solar-pv-rooftop LFSolarPVUtility: - solar-pv-utility + Wind_Onshore: + - onwind estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 1e78725b91..6c6cfc99b9 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -44,8 +44,8 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar-hsat, onwind] - tyndp_renewable_carriers: [solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + renewable_carriers: [solar-hsat] + tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: enable: true @@ -64,6 +64,8 @@ electricity: - solar-pv-rooftop LFSolarPVUtility: - solar-pv-utility + Wind_Onshore: + - onwind estimate_renewable_capacities: # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation From 6afdd2c1265b8f3940532b09893ad41e104fc612 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 16 Jul 2025 11:46:35 +0200 Subject: [PATCH 07/29] fix: check in add existing renwables in case non of the specified carriers are included --- scripts/add_existing_baseyear.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 1eae4a215d..87e3576131 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -111,6 +111,9 @@ def add_existing_renewables( irena = irena.unstack().reset_index() + if not set(tech_map.keys()).intersection(renewable_carriers): + logger.info("No existing capacities to add for specified renewable carriers.") + for carrier, tech in tech_map.items(): if carrier not in renewable_carriers: continue @@ -154,7 +157,9 @@ def add_existing_renewables( df_agg.at[name, "bus"] = bus df_agg.at[name, "resource_class"] = bin_id - df_agg["resource_class"] = df_agg["resource_class"].fillna(0) + df_agg["resource_class"] = ( + df_agg["resource_class"].fillna(0) if "resource_class" in df_agg.columns else 0 + ) def add_power_capacities_installed_before_baseyear( From de41c1b64d910a3a31d87348973f0f02b9365e0e Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 16 Jul 2025 11:47:43 +0200 Subject: [PATCH 08/29] fix: add back onwind and solar to renewable carriers until tyndp carriers are added to the model --- config/config.tyndp.yaml | 2 +- config/test/config.tyndp.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 252b29a201..b5b0234134 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -40,7 +40,7 @@ co2_budget: electricity: base_network: tyndp-raw - renewable_carriers: [solar-hsat, hydro] + renewable_carriers: [onwind, solar, solar-hsat, hydro] tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 6c6cfc99b9..0612de82d2 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -44,7 +44,7 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar-hsat] + renewable_carriers: [onwind, solar, solar-hsat] tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] pecd_renewable_profiles: From 67e2d99dad7ebe3eb0fc025ebb215ff384365f93 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 30 Jul 2025 11:12:18 +0200 Subject: [PATCH 09/29] feat: Slightly improve value error message --- scripts/clean_pecd_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index f839be9ca4..6d322ae3ee 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -133,7 +133,7 @@ def read_pecd_file( if all(data is None for data in pecd): raise ValueError( - f"No PECD data found for {pecd_tech}. Please specify a technology covered within the TYNDP PECD data." + f"No PECD data found for {pecd_tech} in {pyear}. Please specify a technology covered within the TYNDP PECD data." ) pecd_df = pd.concat(pecd, axis=1) fill_na = ( From b487dde76053960713aa7f8d880ee8f228bc67f8 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 30 Jul 2025 11:15:05 +0200 Subject: [PATCH 10/29] feat: add todo for redundant renewable carriers --- config/config.tyndp.yaml | 1 + config/test/config.tyndp.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index b5b0234134..223a4240c6 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -40,6 +40,7 @@ co2_budget: electricity: base_network: tyndp-raw + # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data renewable_carriers: [onwind, solar, solar-hsat, hydro] tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 0612de82d2..b7a684e634 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -44,6 +44,7 @@ electricity: Store: [H2] Link: [H2 pipeline] + # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data renewable_carriers: [onwind, solar, solar-hsat] tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] From f80ac54374c1bd80ce454bb3a5b5308e11e69fa7 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 30 Jul 2025 11:52:39 +0200 Subject: [PATCH 11/29] feat: custom fix for skiprows and typo --- scripts/clean_pecd_data.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 6d322ae3ee..c5eca33d82 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT """ Loads and cleans the available PECD capacity factor generation time series based on PECD weather data. -The script is executed for a given technology, and planning horizon. Technologies can be one of: +The script is executed for a given technology and planning horizon. Technologies can be one of: * CSP * LFSolarPVUtility, @@ -50,9 +50,15 @@ def read_pecd_file( if not os.path.isfile(fn): return None + # Malta CSP data file has an extra header row that must be skipped + if node=="MT00" and technology=="CSP_noStorage" and pyear=="2040": + skiprows=11 + else: + skiprows=10 + pecd_bus = pd.read_csv( fn, - skiprows=10, # first ten rows contain only file metadata + skiprows=skiprows, # first rows contain only file metadata usecols=lambda name: name == "Date" or name == "Hour" or name == str(cyear) From 7793eb7cccc3e9a577dd5ab8c20d2efa63ba1baf Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 30 Jul 2025 11:53:56 +0200 Subject: [PATCH 12/29] feat: add collect rules for clean_pecd_datas and build_renewable_profiles_pecds --- rules/collect.smk | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/rules/collect.smk b/rules/collect.smk index 4c8c25098b..c8d1511bc8 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -81,3 +81,25 @@ rule plot_balance_maps: run=config["run"]["name"], carrier=config_provider("plotting", "balance_map", "bus_carriers")(w), ), + + +rule clean_pecd_datas: + input: + lambda w: + expand( + resources("pecd_data_{technology}_{planning_horizons}.csv"), + **config["scenario"], + run=config["run"]["name"], + technology=config_provider("electricity","pecd_renewable_profiles","technologies")(w), + ) + + +rule build_renewable_profiles_pecds: + input: + lambda w: + expand( + resources("profile_pecd_{clusters}_{technology}.nc"), + **config["scenario"], + run=config["run"]["name"], + technology=config_provider("electricity","pecd_renewable_profiles","technologies")(w), + ) From 956939ae36585b8bd3741c944d069834a59ca6dd Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 30 Jul 2025 11:55:54 +0200 Subject: [PATCH 13/29] feat: add logger warning for missing files --- scripts/clean_pecd_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index c5eca33d82..c412f6f5a8 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -48,6 +48,7 @@ def read_pecd_file( if not os.path.isfile(fn) and "LFSolarPV" in technology: fn = Path(str(fn).replace(technology, "LFSolarPV")) if not os.path.isfile(fn): + logger.warning(f"Missing data for {technology} in {node} in {pyear}.") return None # Malta CSP data file has an extra header row that must be skipped From 0b5a29814d7cc0802f1582881732fcfd17d12fd5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 13:37:47 +0000 Subject: [PATCH 14/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/collect.smk | 32 +++++++++++++++++--------------- scripts/clean_pecd_data.py | 6 +++--- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/rules/collect.smk b/rules/collect.smk index 9bcbff8de0..9783b031a1 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -91,24 +91,26 @@ rule plot_power_networks_clustered: run=config["run"]["name"], ), - + rule clean_pecd_datas: input: - lambda w: - expand( - resources("pecd_data_{technology}_{planning_horizons}.csv"), - **config["scenario"], - run=config["run"]["name"], - technology=config_provider("electricity","pecd_renewable_profiles","technologies")(w), - ) + lambda w: expand( + resources("pecd_data_{technology}_{planning_horizons}.csv"), + **config["scenario"], + run=config["run"]["name"], + technology=config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w), + ), rule build_renewable_profiles_pecds: input: - lambda w: - expand( - resources("profile_pecd_{clusters}_{technology}.nc"), - **config["scenario"], - run=config["run"]["name"], - technology=config_provider("electricity","pecd_renewable_profiles","technologies")(w), - ) + lambda w: expand( + resources("profile_pecd_{clusters}_{technology}.nc"), + **config["scenario"], + run=config["run"]["name"], + technology=config_provider( + "electricity", "pecd_renewable_profiles", "technologies" + )(w), + ), diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index c412f6f5a8..82008f6faf 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -52,10 +52,10 @@ def read_pecd_file( return None # Malta CSP data file has an extra header row that must be skipped - if node=="MT00" and technology=="CSP_noStorage" and pyear=="2040": - skiprows=11 + if node == "MT00" and technology == "CSP_noStorage" and pyear == "2040": + skiprows = 11 else: - skiprows=10 + skiprows = 10 pecd_bus = pd.read_csv( fn, From 405e4da675e9da899ccf14334b92ae81dff29b41 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 30 Jul 2025 17:27:56 +0200 Subject: [PATCH 15/29] feat: introduce safe_pyear helper function --- config/config.tyndp.yaml | 5 ++++ config/test/config.tyndp.yaml | 9 +++++++ rules/build_electricity.smk | 24 ++++++++++++++--- scripts/_helpers.py | 33 ++++++++++++++++++++++++ scripts/build_renewable_profiles_pecd.py | 12 ++++----- scripts/clean_pecd_data.py | 9 +++++-- scripts/clean_tyndp_demand.py | 22 +++++++++------- 7 files changed, 93 insertions(+), 21 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 6504d90c2a..f950c2bf76 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -44,6 +44,11 @@ electricity: pecd_renewable_profiles: enable: true fill_gaps_method: zero + # Complete PECD data is only available for the years 2030, 2040 + # TODO: adjust once udpated 2050 PECD data ist available + available_years: + - 2030 + - 2040 technologies: Wind_Offshore: - offwind-ac-fb-r diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 8f5551142a..4be3e21c00 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -55,6 +55,11 @@ electricity: pecd_renewable_profiles: enable: true fill_gaps_method: zero + # Complete PECD data is only available for the years 2030, 2040 + # TODO: adjust once udpated 2050 PECD data ist available + available_years: + - 2030 + - 2040 technologies: Wind_Offshore: - offwind-ac-fb-r @@ -108,6 +113,10 @@ transmission_projects: load: source: tyndp # opsd, tyndp + available_years_tyndp: + - 2030 + - 2040 + - 2050 fill_gaps: enable: false manual_adjustments: false diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index f06d2ae885..54aec9ddf0 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -2,6 +2,8 @@ # # SPDX-License-Identifier: MIT +from scripts._helpers import safe_pyear + def input_elec_demand(w): return { @@ -400,6 +402,9 @@ rule clean_pecd_data: fill_gaps_method=config_provider( "electricity", "pecd_renewable_profiles", "fill_gaps_method" ), + available_years=config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + ), input: offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", onshore_buses=resources("busmap_base_s_all.csv"), @@ -423,16 +428,26 @@ def input_data_pecd(w): return { f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv") for pyear in set( - config_provider("scenario", "planning_horizons")(w) - ).intersection([2030, 2040]) - # Complete PECD data is only available for the years 2030, 2040 - # TODO: adjust if udpated 2050 data available + [ + safe_pyear( + year, + config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + )(w), + "PECD", + ) + for year in config_provider("scenario", "planning_horizons")(w) + ] + ) } rule build_renewable_profiles_pecd: params: planning_horizons=config_provider("scenario", "planning_horizons"), + available_years=config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + ), input: unpack(input_data_pecd), output: @@ -1034,6 +1049,7 @@ if config["load"]["source"] == "tyndp": planning_horizons=config_provider("scenario", "planning_horizons"), snapshots=config_provider("snapshots"), scenario=config_provider("tyndp_scenario"), + available_years=config_provider("load", "available_years_tyndp"), input: electricity_demand=directory("data/tyndp_2024_bundle/Demand Profiles"), output: diff --git a/scripts/_helpers.py b/scripts/_helpers.py index f1538ccca4..55ede2690d 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1129,3 +1129,36 @@ def extract_grid_data_tyndp( links.index = links.apply(make_index, axis=1, prefix=carrier) return links + + +def safe_pyear(year: int, available_years: list = [2030, 2040, 2050], source="TYNDP"): + """ + Checks and adjusts whether a given pyear is in the available years and falls back to the previous available year. + + Parameters + ---------- + year : int + planning horizon year which will be checked and possibly adjusted to previous available year + available_years : list + list of available years + source : str, optional + source of the data for which availability will be checked. Defaults to "TYNDP" + + Returns + ------- + year_new : str + safe pyear as a string + """ + + if not available_years: + raise ValueError("`available_years` cannot be empty.") + if year not in available_years: + lower = [y for y in available_years if y < year] + year_new = max(lower) if lower else available_years[0] + logger.warning( + f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}." + ) + else: + year_new = year + + return str(year_new) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 67646349b6..f4dcd5d3c8 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -22,12 +22,12 @@ import logging -import numpy as np import pandas as pd import xarray as xr from scripts._helpers import ( configure_logging, + safe_pyear, set_scenario_config, ) @@ -55,11 +55,11 @@ f"Extract PECD capacity factor time series for year {year} for technology {technology}..." ) year_i = year - if int(year) not in [2030, 2040, 2050]: - year = np.clip(10 * (year // 10), 2030, 2050) - logger.warning( - f"TYNDP PECD data unavailable for planning horizon. Falling back to previous available year {year}." - ) + # falling back to latest available pyear if not in list of available years + year = safe_pyear( + int(year), available_years=snakemake.params.available_years, source="PECD" + ) + # TODO: remove once PECD data is updated if year == 2050: logger.warning( "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data." diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 82008f6faf..562fe63bae 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -28,6 +28,7 @@ from scripts._helpers import ( configure_logging, get_snapshots, + safe_pyear, set_scenario_config, ) @@ -102,8 +103,12 @@ def read_pecd_file( ) cyear = 2009 - # Planning year - pyear = str(snakemake.wildcards.planning_horizons) + # Planning year (falls back to latest available pyear if not in list of available years) + pyear = safe_pyear( + int(snakemake.wildcards.planning_horizons), + available_years=snakemake.params.available_years, + source="PECD", + ) # Technology as in PECD terminology pecd_tech = snakemake.wildcards.technology diff --git a/scripts/clean_tyndp_demand.py b/scripts/clean_tyndp_demand.py index 3a03c391f5..72d34f2a51 100644 --- a/scripts/clean_tyndp_demand.py +++ b/scripts/clean_tyndp_demand.py @@ -12,15 +12,21 @@ from functools import partial from pathlib import Path -import numpy as np import pandas as pd -from _helpers import configure_logging, get_snapshots, set_scenario_config +from _helpers import ( + configure_logging, + get_snapshots, + safe_pyear, + set_scenario_config, +) from tqdm import tqdm logger = logging.getLogger(__name__) -def load_elec_demand(fn: str, scenario: str, pyear: int, cyear: int): +def load_elec_demand( + fn: str, scenario: str, pyear: int, cyear: int, available_years: list +): """ Load electricity demand files into dictionary of dataframes. Filter for specific climatic year and format data. """ @@ -28,12 +34,9 @@ def load_elec_demand(fn: str, scenario: str, pyear: int, cyear: int): # handle intermediate years # TODO: Possibly improve this with linear interpolation for 2035 and 2045 - if pyear not in [2030, 2040, 2050]: - pyear = np.clip(10 * (pyear // 10), 2030, 2050) - logger.warning( - "Planning horizon doesn't match available 2024 TYNDP electricity demand data. " - f"Falling back to previous available year {pyear}." - ) + pyear = int( + safe_pyear(int(pyear), available_years=available_years, source="TYNDP demand") + ) if scenario == "NT": if pyear == 2050: logger.warning( @@ -132,6 +135,7 @@ def load_elec_demand(fn: str, scenario: str, pyear: int, cyear: int): snakemake.input.electricity_demand, scenario, cyear=cyear, + available_years=snakemake.params.available_years, ) with mp.Pool(processes=snakemake.threads) as pool: From 96794be1e8537a6a3d72c4b507bb48f2db7a7d9c Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 30 Jul 2025 17:57:57 +0200 Subject: [PATCH 16/29] feat: make handling of GB and UK more consistent --- scripts/clean_pecd_data.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 562fe63bae..27eb7faa62 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -43,7 +43,11 @@ def read_pecd_file( technology: str, sns: pd.DatetimeIndex, ): - fn = Path(dir_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + fn = Path( + dir_pecd, + pyear, + f"PECD_{technology}_{pyear}_{node.replace('GB', 'UK')}_edition 2023.2.csv", + ) # PECD only differentiates between utility and rooftop PV for some nodes if not os.path.isfile(fn) and "LFSolarPV" in technology: @@ -117,9 +121,11 @@ def read_pecd_file( onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) nodes = ( - offshore_buses.index + offshore_buses.index.str.replace( + "UK", "GB", regex=True + ) # replace UK with GB for naming convention if pecd_tech == "Wind_Offshore" - else onshore_buses.index.str.replace("GB", "UK", regex=True) + else onshore_buses.index ) dir_pecd = snakemake.input.dir_pecd @@ -154,11 +160,9 @@ def read_pecd_file( else pecd_df.agg(snakemake.params.fill_gaps_method, axis=1) ) pecd_df = ( - pecd_df.reindex(nodes, axis=1) # include missing node data with empty columns - .rename( - columns=lambda x: x.replace("UK", "GB") - ) # replace UK with GB for naming convention - .where( + pecd_df.reindex( + nodes, axis=1 + ).where( # include missing node data with empty columns lambda df: df.notna(), fill_na, axis=0 ) # fill missing node data with configured aggregation method ) From 35e4d45a16a59f3ad5e29bedd2dc0fc2d61f692c Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Wed, 30 Jul 2025 18:02:50 +0200 Subject: [PATCH 17/29] feat: adjust wording of logger about existing capacities --- scripts/add_existing_baseyear.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 06308f8a7e..0563c3589f 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -103,7 +103,10 @@ def add_existing_renewables( irena = irena.unstack().reset_index() if not set(tech_map.keys()).intersection(renewable_carriers): - logger.info("No existing capacities to add for specified renewable carriers.") + # TODO: adjust once TYNDP existing capacities are added. + logger.info( + "No existing capacities to add for specified renewable carriers. Existing capacities of TYNDP renewable carriers will be added separately." + ) for carrier, tech in tech_map.items(): if carrier not in renewable_carriers: From a78fbc57e9ed5514c846904bd874868ed5b60fe2 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 31 Jul 2025 11:21:09 +0200 Subject: [PATCH 18/29] doc: document compatibility with Solar CSP --- doc/configtables/electricity.csv | 2 +- scripts/clean_pecd_data.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 29ef77ed30..0958e54dac 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -32,7 +32,7 @@ conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv-rooftop, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh}",List of TYNDP renewable generators to include in the model. Technologies covered by specified TYNDP renewable carriers need to be removed from `estimate_renewable_carriers` technology list. pecd_renewable_profiles,,, --- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPVUtility, LFSolarPVRooftop, Wind_Offshore, Wind_Onshore}. +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {LFSolarPVUtility, LFSolarPVRooftop, Wind_Offshore, Wind_Onshore, CSP_noStorage, CSP_withStorage_7h_dispatched, CSP_withStorage_7h_preDispatch}. -- fill_gaps_method,,str,The chosen method for filling gaps in the PECD data to the modelled nodes. Can be either `zero` to fill with zero values or any other aggregation method such as `mean`, `median`, `max` or similar. -- technologies,,, -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 27eb7faa62..600dbc449e 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -5,11 +5,13 @@ Loads and cleans the available PECD capacity factor generation time series based on PECD weather data. The script is executed for a given technology and planning horizon. Technologies can be one of: - * CSP * LFSolarPVUtility, * LFSolarPVRooftop, * Wind_Offshore, - * Wind_Onshore. + * Wind_Onshore, + * CSP_noStorage, + * CSP_withStorage_7h_dispatched, + * CSP_withStorage_7h_preDispatch (note: includes cf > 1 for when thermal storage can be used). Outputs ------- From 7bbd23dd0d8ba4b4d051cc9946e528f797ade974 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 31 Jul 2025 11:35:48 +0200 Subject: [PATCH 19/29] doc: document new config option for available pyears in all configs and documentation files --- config/config.default.yaml | 8 ++++++++ config/config.tyndp.yaml | 4 ++++ config/test/config.tyndp.yaml | 4 ++-- doc/configtables/electricity.csv | 1 + doc/configtables/load.csv | 1 + 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index ee457d5fe8..393a129bb1 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -145,6 +145,10 @@ electricity: pecd_renewable_profiles: enable: false fill_gaps_method: zero + available_years: + - 2030 + - 2040 + - 2050 technologies: Wind_Offshore: - offwind-ac-fb-r @@ -393,6 +397,10 @@ load: enable: true interpolate_limit: 3 time_shift_for_large_gaps: 1w + available_years_tyndp: + - 2030 + - 2040 + - 2050 manual_adjustments: true scaling_factor: 1.0 fixed_year: false diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index f950c2bf76..a97fe8de57 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -89,6 +89,10 @@ load: source: tyndp # opsd, tyndp fill_gaps: enable: false + available_years_tyndp: + - 2030 + - 2040 + - 2050 manual_adjustments: false supplement_synthetic: false diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 4be3e21c00..65ed437cfd 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -113,12 +113,12 @@ transmission_projects: load: source: tyndp # opsd, tyndp + fill_gaps: + enable: false available_years_tyndp: - 2030 - 2040 - 2050 - fill_gaps: - enable: false manual_adjustments: false supplement_synthetic: false diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 0958e54dac..6ce61b39a7 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -34,6 +34,7 @@ tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv pecd_renewable_profiles,,, -- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {LFSolarPVUtility, LFSolarPVRooftop, Wind_Offshore, Wind_Onshore, CSP_noStorage, CSP_withStorage_7h_dispatched, CSP_withStorage_7h_preDispatch}. -- fill_gaps_method,,str,The chosen method for filling gaps in the PECD data to the modelled nodes. Can be either `zero` to fill with zero values or any other aggregation method such as `mean`, `median`, `max` or similar. +-- available_years,--,list,"List of years for which PECD data is available." -- technologies,,, -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles. diff --git a/doc/configtables/load.csv b/doc/configtables/load.csv index 79e75f2029..68e5ce4dbf 100644 --- a/doc/configtables/load.csv +++ b/doc/configtables/load.csv @@ -4,6 +4,7 @@ fill_gaps,--,--,"Gaps filling strategy used." -- enable,bool,"{true, false}","Whether to fill gaps using interpolation for small gaps and time shift for large gaps." -- interpolate_limit,hours,integer,"Maximum gap size (consecutive nans) which interpolated linearly." -- time_shift_for_large_gaps,string,string,"Periods which are used for copying time-slices in order to fill large gaps of nans. Have to be valid ``pandas`` period strings." +available_years_tyndp,--,list,"List of years for which TYNDP demand data is available." manual_adjustments,bool,"{true, false}","Whether to adjust the load data manually according to the function in :func:`manual_adjustment`." scaling_factor,--,float,"Global correction factor for the load time series." fixed_year,--,Year or False,"To specify a fixed year for the load time series that deviates from the snapshots' year" From 2e274a106ff25b89261fb0522e1d9f21ccc4016c Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 31 Jul 2025 11:49:30 +0200 Subject: [PATCH 20/29] doc: add release note --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 68a3873d2c..4ae7f5151c 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -11,7 +11,7 @@ Release Notes Upcoming Open-TYNDP Release ================ - +* Add complete compatibility for processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP), replacing default ERA5- and SARAH3-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/71). This implementation serves to facilitate a sub-workflow for creation of the renewable profiles but does not yet attach them to any technologies yet. Upcoming PyPSA-Eur Release From 27abc957a19ad3f3bde411f041050b4cef2b0922 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Thu, 31 Jul 2025 14:27:54 +0200 Subject: [PATCH 21/29] feat: add verbose option to safe_pyear helper function --- rules/build_electricity.smk | 1 + scripts/_helpers.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 54aec9ddf0..3f43bb6188 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -435,6 +435,7 @@ def input_data_pecd(w): "electricity", "pecd_renewable_profiles", "available_years" )(w), "PECD", + verbose=False, ) for year in config_provider("scenario", "planning_horizons")(w) ] diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 55ede2690d..16936bd86d 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1131,7 +1131,12 @@ def extract_grid_data_tyndp( return links -def safe_pyear(year: int, available_years: list = [2030, 2040, 2050], source="TYNDP"): +def safe_pyear( + year: int, + available_years: list = [2030, 2040, 2050], + source: str = "TYNDP", + verbose: bool = True, +): """ Checks and adjusts whether a given pyear is in the available years and falls back to the previous available year. @@ -1139,10 +1144,12 @@ def safe_pyear(year: int, available_years: list = [2030, 2040, 2050], source="TY ---------- year : int planning horizon year which will be checked and possibly adjusted to previous available year - available_years : list - list of available years + available_years : list, optional + list of available years. Defaults to [2030, 2040, 2050] source : str, optional source of the data for which availability will be checked. Defaults to "TYNDP" + verbose : bool, optional + Whether to activate verbose logging. Defaults to True Returns ------- @@ -1155,9 +1162,10 @@ def safe_pyear(year: int, available_years: list = [2030, 2040, 2050], source="TY if year not in available_years: lower = [y for y in available_years if y < year] year_new = max(lower) if lower else available_years[0] - logger.warning( - f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}." - ) + if verbose: + logger.warning( + f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}." + ) else: year_new = year From cf4fc7ee909bcaffe2a44f5260d5103fc2099c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20R=C3=BCdt?= <117752024+daniel-rdt@users.noreply.github.com> Date: Fri, 1 Aug 2025 15:34:20 +0200 Subject: [PATCH 22/29] Apply suggestions from code review Co-authored-by: Thomas Gilon --- config/config.tyndp.yaml | 2 +- config/test/config.tyndp.yaml | 2 +- doc/release_notes.rst | 2 +- scripts/_helpers.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index a97fe8de57..98000d015b 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -45,7 +45,7 @@ electricity: enable: true fill_gaps_method: zero # Complete PECD data is only available for the years 2030, 2040 - # TODO: adjust once udpated 2050 PECD data ist available + # TODO: adjust once udpated 2050 PECD data is available available_years: - 2030 - 2040 diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 65ed437cfd..d23af674c0 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -56,7 +56,7 @@ electricity: enable: true fill_gaps_method: zero # Complete PECD data is only available for the years 2030, 2040 - # TODO: adjust once udpated 2050 PECD data ist available + # TODO: adjust once udpated 2050 PECD data is available available_years: - 2030 - 2040 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 4ae7f5151c..e4f37d392b 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -11,7 +11,7 @@ Release Notes Upcoming Open-TYNDP Release ================ -* Add complete compatibility for processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP), replacing default ERA5- and SARAH3-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/71). This implementation serves to facilitate a sub-workflow for creation of the renewable profiles but does not yet attach them to any technologies yet. +* Add complete compatibility for processing and preparation of PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP) (https://github.com/open-energy-transition/open-tyndp/pull/71). These profiles are used for the TYNDP 2024 and replace the default ERA5- and SARAH3-based profiles processed with Atlite. This implementation serves to facilitate a sub-workflow for creation of the renewable profiles, but does not yet attach them to any technologies. Upcoming PyPSA-Eur Release diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 16936bd86d..a46df92f25 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1138,14 +1138,14 @@ def safe_pyear( verbose: bool = True, ): """ - Checks and adjusts whether a given pyear is in the available years and falls back to the previous available year. + Checks and adjusts whether a given pyear is in the available years of a given data source. If not, it falls back to the previous available year. Parameters ---------- year : int planning horizon year which will be checked and possibly adjusted to previous available year available_years : list, optional - list of available years. Defaults to [2030, 2040, 2050] + List of available years. Defaults to [2030, 2040, 2050] source : str, optional source of the data for which availability will be checked. Defaults to "TYNDP" verbose : bool, optional @@ -1158,7 +1158,7 @@ def safe_pyear( """ if not available_years: - raise ValueError("`available_years` cannot be empty.") + raise ValueError("No `available_years` provided. Expected a non-empty list of years.") if year not in available_years: lower = [y for y in available_years if y < year] year_new = max(lower) if lower else available_years[0] From 7e51535c4010f227c0ce652cb1fa43a38c8502ed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 13:35:33 +0000 Subject: [PATCH 23/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index a46df92f25..5e0854b25f 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1158,7 +1158,9 @@ def safe_pyear( """ if not available_years: - raise ValueError("No `available_years` provided. Expected a non-empty list of years.") + raise ValueError( + "No `available_years` provided. Expected a non-empty list of years." + ) if year not in available_years: lower = [y for y in available_years if y < year] year_new = max(lower) if lower else available_years[0] From 67f44acfc06e19a02b805599e988bd27a606dc1e Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 1 Aug 2025 15:37:31 +0200 Subject: [PATCH 24/29] fix: remove manual 2050 fallback to 2040 for PECD data --- scripts/build_renewable_profiles_pecd.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index f4dcd5d3c8..51ff644352 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -59,12 +59,6 @@ year = safe_pyear( int(year), available_years=snakemake.params.available_years, source="PECD" ) - # TODO: remove once PECD data is updated - if year == 2050: - logger.warning( - "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data." - ) - year = 2040 profile = ( pd.read_csv( From 7c0a0c00de3be18b456a9ddd540e400e74b18f1f Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Fri, 1 Aug 2025 16:23:49 +0200 Subject: [PATCH 25/29] feat: adjust input and output types in safe_pyear --- scripts/_helpers.py | 18 ++++++++++-------- scripts/build_renewable_profiles_pecd.py | 2 +- scripts/clean_pecd_data.py | 8 ++++---- scripts/clean_tyndp_demand.py | 5 ++--- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 5e0854b25f..de46230877 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1132,7 +1132,7 @@ def extract_grid_data_tyndp( def safe_pyear( - year: int, + year: int | str, available_years: list = [2030, 2040, 2050], source: str = "TYNDP", verbose: bool = True, @@ -1143,24 +1143,26 @@ def safe_pyear( Parameters ---------- year : int - planning horizon year which will be checked and possibly adjusted to previous available year + planning horizon year which will be checked and possibly adjusted to previous available year. available_years : list, optional - List of available years. Defaults to [2030, 2040, 2050] + List of available years. Defaults to [2030, 2040, 2050]. source : str, optional - source of the data for which availability will be checked. Defaults to "TYNDP" + Source of the data for which availability will be checked. For logging purpose only. Defaults to "TYNDP". verbose : bool, optional - Whether to activate verbose logging. Defaults to True + Whether to activate verbose logging. Defaults to True. Returns ------- - year_new : str - safe pyear as a string + year_new : int + safe pyear adjusted for available years """ if not available_years: raise ValueError( "No `available_years` provided. Expected a non-empty list of years." ) + if not isinstance(year, int): + year = int(year) if year not in available_years: lower = [y for y in available_years if y < year] year_new = max(lower) if lower else available_years[0] @@ -1171,4 +1173,4 @@ def safe_pyear( else: year_new = year - return str(year_new) + return year_new diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py index 51ff644352..0fbcc8c410 100644 --- a/scripts/build_renewable_profiles_pecd.py +++ b/scripts/build_renewable_profiles_pecd.py @@ -57,7 +57,7 @@ year_i = year # falling back to latest available pyear if not in list of available years year = safe_pyear( - int(year), available_years=snakemake.params.available_years, source="PECD" + year, available_years=snakemake.params.available_years, source="PECD" ) profile = ( diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py index 600dbc449e..2f8f6567ee 100644 --- a/scripts/clean_pecd_data.py +++ b/scripts/clean_pecd_data.py @@ -41,13 +41,13 @@ def read_pecd_file( node: str, dir_pecd: str, cyear: str, - pyear: str, + pyear: int, technology: str, sns: pd.DatetimeIndex, ): fn = Path( dir_pecd, - pyear, + str(pyear), f"PECD_{technology}_{pyear}_{node.replace('GB', 'UK')}_edition 2023.2.csv", ) @@ -59,7 +59,7 @@ def read_pecd_file( return None # Malta CSP data file has an extra header row that must be skipped - if node == "MT00" and technology == "CSP_noStorage" and pyear == "2040": + if node == "MT00" and technology == "CSP_noStorage" and pyear == 2040: skiprows = 11 else: skiprows = 10 @@ -111,7 +111,7 @@ def read_pecd_file( # Planning year (falls back to latest available pyear if not in list of available years) pyear = safe_pyear( - int(snakemake.wildcards.planning_horizons), + snakemake.wildcards.planning_horizons, available_years=snakemake.params.available_years, source="PECD", ) diff --git a/scripts/clean_tyndp_demand.py b/scripts/clean_tyndp_demand.py index 72d34f2a51..24ee5479b8 100644 --- a/scripts/clean_tyndp_demand.py +++ b/scripts/clean_tyndp_demand.py @@ -34,9 +34,8 @@ def load_elec_demand( # handle intermediate years # TODO: Possibly improve this with linear interpolation for 2035 and 2045 - pyear = int( - safe_pyear(int(pyear), available_years=available_years, source="TYNDP demand") - ) + pyear = safe_pyear(pyear, available_years=available_years, source="TYNDP demand") + if scenario == "NT": if pyear == 2050: logger.warning( From 17c880cdf15eb6aa2c841d80415f45c8dddc2e8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20R=C3=BCdt?= <117752024+daniel-rdt@users.noreply.github.com> Date: Fri, 1 Aug 2025 17:36:54 +0200 Subject: [PATCH 26/29] Apply suggestions from code review Co-authored-by: Thomas Gilon --- scripts/_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index de46230877..e6a7466176 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1143,7 +1143,7 @@ def safe_pyear( Parameters ---------- year : int - planning horizon year which will be checked and possibly adjusted to previous available year. + Planning horizon year which will be checked and possibly adjusted to previous available year. available_years : list, optional List of available years. Defaults to [2030, 2040, 2050]. source : str, optional @@ -1154,7 +1154,7 @@ def safe_pyear( Returns ------- year_new : int - safe pyear adjusted for available years + Safe pyear adjusted for available years """ if not available_years: From cedc093827e6840225b8825a44dc2c75235200b0 Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 11 Aug 2025 11:04:59 +0200 Subject: [PATCH 27/29] feat: refactor input_data_pecd function for easier readability --- rules/build_electricity.smk | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 3f43bb6188..7f5d7b3a36 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -425,21 +425,17 @@ rule clean_pecd_data: def input_data_pecd(w): + available_years = config_provider( + "electricity", "pecd_renewable_profiles", "available_years" + )(w) + planning_horizons = config_provider("scenario", "planning_horizons")(w) + safe_pyears = set( + safe_pyear(year, available_years, "PECD", verbose=False) + for year in planning_horizons + ) return { f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv") - for pyear in set( - [ - safe_pyear( - year, - config_provider( - "electricity", "pecd_renewable_profiles", "available_years" - )(w), - "PECD", - verbose=False, - ) - for year in config_provider("scenario", "planning_horizons")(w) - ] - ) + for pyear in safe_pyears } From 0cb02f8cfba3e5619b2762491e2eac6958feba5a Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 11 Aug 2025 11:57:13 +0200 Subject: [PATCH 28/29] apply suggestions from code review --- scripts/_helpers.py | 6 ++++-- scripts/add_existing_baseyear.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index e6a7466176..ff4e3a7f92 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -9,6 +9,7 @@ import os import re import time +from bisect import bisect_right from functools import partial, wraps from pathlib import Path from tempfile import NamedTemporaryFile @@ -1164,8 +1165,9 @@ def safe_pyear( if not isinstance(year, int): year = int(year) if year not in available_years: - lower = [y for y in available_years if y < year] - year_new = max(lower) if lower else available_years[0] + year_new = available_years[ + bisect_right(sorted(available_years), year, lo=1) - 1 + ] if verbose: logger.warning( f"{source} data unavailable for planning horizon {year}. Falling back to previous available year {year_new}." diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 0563c3589f..68c2a55960 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -102,7 +102,7 @@ def add_existing_renewables( irena = irena.unstack().reset_index() - if not set(tech_map.keys()).intersection(renewable_carriers): + if not set(tech_map).intersection(renewable_carriers): # TODO: adjust once TYNDP existing capacities are added. logger.info( "No existing capacities to add for specified renewable carriers. Existing capacities of TYNDP renewable carriers will be added separately." From 90616eafcd526053936da0e34f398fcf3849326d Mon Sep 17 00:00:00 2001 From: daniel-rdt Date: Mon, 11 Aug 2025 12:26:56 +0200 Subject: [PATCH 29/29] doc: add documentation for retrieve_tyndp_pecd_data rule --- doc/data-retrieval.rst | 8 ++++++++ doc/retrieve.rst | 5 +++++ scripts/retrieve_tyndp_pecd_data.py | 2 ++ 3 files changed, 15 insertions(+) diff --git a/doc/data-retrieval.rst b/doc/data-retrieval.rst index 5ed4bb51a1..c6af8dd9c6 100644 --- a/doc/data-retrieval.rst +++ b/doc/data-retrieval.rst @@ -243,3 +243,11 @@ Data in this section is retrieved and extracted in rules specified in ``rules/re - **Link:** https://cdn.jsdelivr.net/gh/gavinr/world-countries-centroids@v1.0.0/dist/countries.geojson - **License:** MIT License - **Description:** Contains centroids of world countries in GeoJson. + +``data/tyndp_2024_bundle/PECD`` + +- **Source:** TYNDP 2024 Scenarios, https://2024.entsos-tyndp-scenarios.eu/download/ +- **Link:** https://storage.googleapis.com/open-tyndp-data-store/PECD.zip +- **License:** CC-BY 4.0 +- **Description:** Contains hourly PECD v3.1 capacity factor time series for Solar PV rooftop, + Solar PV utility, Onshore Wind, Offshore Wind and Solar CSP. diff --git a/doc/retrieve.rst b/doc/retrieve.rst index 40f4fa7469..378d43727b 100644 --- a/doc/retrieve.rst +++ b/doc/retrieve.rst @@ -167,3 +167,8 @@ None. **Outputs** - ``data/countries_centroids.geojson`` + +Rule ``retrieve_tyndp_pecd_data`` +==================================== + +.. automodule:: retrieve_tyndp_pecd_data diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py index 8dc9ffdb1b..8d1666da7a 100644 --- a/scripts/retrieve_tyndp_pecd_data.py +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -6,6 +6,8 @@ This rule downloads the TYNDP PECD v3.1 data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` subdirectory, such that all files of the TYNDP bundle are stored in it. +The original data is published by ENTSO-E and ENTSOG under Creative Commons Attribution 4.0 International License (CC-BY 4.0) +and can be found under https://2024.entsos-tyndp-scenarios.eu/download/. **Outputs**