diff --git a/Snakefile b/Snakefile index f21e0c5568..a6c449205a 100644 --- a/Snakefile +++ b/Snakefile @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: MIT +from itertools import chain from pathlib import Path import yaml from os.path import normpath, exists, join diff --git a/config/config.default.yaml b/config/config.default.yaml index 9f54ecff48..16343a5c39 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -112,12 +112,30 @@ electricity: conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float, hydro] + tyndp_renewable_carriers: [] + + pecd_renewable_profiles: + enable: false + technologies: + Wind_Offshore: + - offwind-ac-fb-r + - offwind-ac-fl-r + - offwind-dc-fb-r + - offwind-dc-fl-r + - offwind-dc-fb-oh + - offwind-dc-fl-oh + - offwind-h2-fb-oh + - offwind-h2-fl-oh estimate_renewable_capacities: enable: true from_gem: true year: 2020 expansion_limit: false + technologies: + - Offshore + - Onshore + - PV technology_mapping: Offshore: offwind-ac Onshore: onwind diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 3fab9b45e9..45c07636fd 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -39,6 +39,29 @@ co2_budget: electricity: base_network: tyndp-raw + + renewable_carriers: [solar, solar-hsat, onwind, hydro] + tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + + pecd_renewable_profiles: + enable: true + technologies: + Wind_Offshore: + - offwind-ac-fb-r + - offwind-ac-fl-r + - offwind-dc-fb-r + - offwind-dc-fl-r + - offwind-dc-fb-oh + - offwind-dc-fl-oh + - offwind-h2-fb-oh + - offwind-h2-fl-oh + + estimate_renewable_capacities: + # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation + technologies: + - Onshore + - PV + transmission_limit: v1.0 links: diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index 0a91c02a9f..9e781ec6e5 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -37,7 +37,6 @@ co2_budget: electricity: base_network: tyndp-raw - transmission_limit: v1.0 extendable_carriers: Generator: [OCGT] @@ -45,7 +44,29 @@ electricity: Store: [H2] Link: [H2 pipeline] - renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float] + renewable_carriers: [solar, solar-hsat, onwind] + tyndp_renewable_carriers: [offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh] + + pecd_renewable_profiles: + enable: true + technologies: + Wind_Offshore: + - offwind-ac-fb-r + - offwind-ac-fl-r + - offwind-dc-fb-r + - offwind-dc-fl-r + - offwind-dc-fb-oh + - offwind-dc-fl-oh + - offwind-h2-fb-oh + - offwind-h2-fl-oh + + estimate_renewable_capacities: + # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation + technologies: + - Onshore + - PV + + transmission_limit: v1.0 atlite: default_cutout: europe-2013-03-sarah3-era5 diff --git a/doc/configtables/electricity.csv b/doc/configtables/electricity.csv index 4ea09ef663..2e8a61e495 100644 --- a/doc/configtables/electricity.csv +++ b/doc/configtables/electricity.csv @@ -1,45 +1,52 @@ -,Unit,Values,Description -voltages,kV,"Any subset of {220., 300., 330., 380., 400., 500., 750.}",Voltage levels to consider -base_network, --, "Any value in {'entsoegridkit', 'osm-prebuilt', 'osm-raw}", "Specify the underlying base network, i.e. GridKit (based on ENTSO-E web map extract, OpenStreetMap (OSM) prebuilt or raw (built from raw OSM data), takes longer." -osm-prebuilt-version, --, "float, any value in range 0.1-0.6", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release." -gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``. -gaslimit,MWhth,float or false,Global gas usage limit -co2limit_enable,bool,true or false,"Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit`` in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks." -co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions -co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard. -operational_reserve,,,Settings for reserve requirements following `GenX `_ -,,, --- activate,bool,true or false,Whether to take operational reserve requirements into account during optimisation --- epsilon_load,--,float,share of total load --- epsilon_vres,--,float,share of total renewable supply --- contingency,MW,float,fixed reserve capacity -max_hours,,, --- battery,h,float,Maximum state of charge capacity of the battery in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. --- H2,h,float,Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. -extendable_carriers,,, --- Generator,--,Any extendable carrier,"Defines existing or non-existing conventional and renewable power plants to be extendable during the optimization. Conventional generators can only be built/expanded where already existent today. If a listed conventional carrier is not included in the ``conventional_carriers`` list, the lower limit of the capacity expansion is set to 0." --- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. --- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. --- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``. -powerplants_filter,--,"use `pandas.query `_ strings here, e.g. ``Country not in ['Germany']``",Filter query for the default powerplant database. -,,, -custom_powerplants,--,"use `pandas.query `_ strings here, e.g. ``Country in ['Germany']``",Filter query for the custom powerplant database. -,,, -everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to add to every node in the model with zero initial capacity. To be used in combination with ``extendable_carriers`` to allow for building conventional powerplants irrespective of existing locations." -,,, -conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound." -,,, -renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. -estimate_renewable_capacities,,, --- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. --- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. --- year,--,bool,Renewable capacities are based on existing capacities reported by IRENA (IRENASTAT) for the specified year --- expansion_limit,--,float or false,"Artificially limit maximum IRENA capacities to a factor. For example, an ``expansion_limit: 1.1`` means 110% of capacities . If false are chosen, the estimated renewable potentials determine by the workflow are used." --- technology_mapping,,,Mapping between PyPSA-Eur and powerplantmatching technology names --- -- Offshore,--,"{onwind}","PyPSA-Eur carrier that is considered for existing onshore wind capacities (IRENA, GEM)." --- -- Offshore,--,"Any of {offwind-ac, offwind-dc, offwind-float}","PyPSA-Eur carrier that is considered for existing offshore wind technology (IRENA, GEM)." --- -- PV,--,{solar},"PyPSA-Eur carrier that is considered for existing solar PV capacities (IRENA, GEM)." -autarky,,, --- enable,bool,true or false,Require each node to be autarkic by removing all lines and links. --- by_country,bool,true or false,Require each country to be autarkic by removing all cross-border lines and links. ``electricity: autarky`` must be enabled. -transmission_limit,str,"Values like 'vopt', 'v1.25', 'copt', 'c1.25'","Limit on transmission expansion. The first part can be ``v`` (for setting a limit on line volume) or ``c`` (for setting a limit on line cost). The second part can be ``opt`` or a float bigger than one (e.g. 1.25). If ``opt`` is chosen line expansion is optimised according to its capital cost (where the choice ``v`` only considers overhead costs for HVDC transmission lines, while ``c`` uses more accurate costs distinguishing between overhead and underwater sections and including inverter pairs). The setting ``v1.25`` will limit the total volume of line expansion to 25% of currently installed capacities weighted by individual line lengths. The setting ``c1.25`` will allow to build a transmission network that costs no more than 25 % more than the current system." +,Unit,Values,Description +voltages,kV,"Any subset of {220., 300., 330., 380., 400., 500., 750.}",Voltage levels to consider +base_network, --, "Any value in {'entsoegridkit', 'osm-prebuilt', 'osm-raw}", "Specify the underlying base network, i.e. GridKit (based on ENTSO-E web map extract, OpenStreetMap (OSM) prebuilt or raw (built from raw OSM data), takes longer." +osm-prebuilt-version, --, "float, any value in range 0.1-0.6", "Choose the version of the prebuilt OSM network. Defaults to latest Zenodo release." +gaslimit_enable,bool,true or false,Add an overall absolute gas limit configured in ``electricity: gaslimit``. +gaslimit,MWhth,float or false,Global gas usage limit +co2limit_enable,bool,true or false,"Add an overall absolute carbon-dioxide emissions limit configured in ``electricity: co2limit`` in :mod:`prepare_network`. **Warning:** This option should currently only be used with electricity-only networks, not for sector-coupled networks." +co2limit,:math:`t_{CO_2-eq}/a`,float,Cap on total annual system carbon dioxide emissions +co2base,:math:`t_{CO_2-eq}/a`,float,Reference value of total annual system carbon dioxide emissions if relative emission reduction target is specified in ``{opts}`` wildcard. +operational_reserve,,,Settings for reserve requirements following `GenX `_ +,,, +-- activate,bool,true or false,Whether to take operational reserve requirements into account during optimisation +-- epsilon_load,--,float,share of total load +-- epsilon_vres,--,float,share of total renewable supply +-- contingency,MW,float,fixed reserve capacity +max_hours,,, +-- battery,h,float,Maximum state of charge capacity of the battery in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. +-- H2,h,float,Maximum state of charge capacity of the hydrogen storage in terms of hours at full output capacity ``p_nom``. Cf. `PyPSA documentation `_. +extendable_carriers,,, +-- Generator,--,Any extendable carrier,"Defines existing or non-existing conventional and renewable power plants to be extendable during the optimization. Conventional generators can only be built/expanded where already existent today. If a listed conventional carrier is not included in the ``conventional_carriers`` list, the lower limit of the capacity expansion is set to 0." +-- StorageUnit,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. +-- Store,--,"Any subset of {'battery','H2'}",Adds extendable storage units (battery and/or hydrogen) at every node/bus after clustering without capacity limits and with zero initial capacity. +-- Link,--,Any subset of {'H2 pipeline'},Adds extendable links (H2 pipelines only) at every connection where there are lines or HVDC links without capacity limits and with zero initial capacity. Hydrogen pipelines require hydrogen storage to be modelled as ``Store``. +powerplants_filter,--,"use `pandas.query `_ strings here, e.g. ``Country not in ['Germany']``",Filter query for the default powerplant database. +,,, +custom_powerplants,--,"use `pandas.query `_ strings here, e.g. ``Country in ['Germany']``",Filter query for the custom powerplant database. +,,, +everywhere_powerplants,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to add to every node in the model with zero initial capacity. To be used in combination with ``extendable_carriers`` to allow for building conventional powerplants irrespective of existing locations." +,,, +conventional_carriers,--,"Any subset of {nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass}","List of conventional power plants to include in the model from ``resources/powerplants_s_{clusters}.csv``. If an included carrier is also listed in ``extendable_carriers``, the capacity is taken as a lower bound." +,,, +renewable_carriers,--,"Any subset of {solar, onwind, offwind-ac, offwind-dc, offwind-float, hydro}",List of renewable generators to include in the model. +tyndp_renewable_carriers,--,"Any subset of {solar-pv, solar-pv-utility, solar-pv-rooftop, onwind, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh}",List of TYNDP renewable generators to include in the model. Technologies covered by specified TYNDP renewable carriers need to be removed from `estimate_renewable_carriers` technology list. +pecd_renewable_profiles,,, +-- enable,,bool,Activate PECD renewable profiles from 2024 TYNDP instead of default renewable profiles for specified renewable technologies below. Technologies ``pecd_techs`` can be any of {CSP, LFSolarPV, Wind_Offshore, Wind_Onshore}. +-- technologies,,, +-- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``. +-- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles. +estimate_renewable_capacities,,, +-- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`. +-- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker `_ and `Global Energy Monitor's Global Wind Power Tracker `_. +-- year,--,bool,Renewable capacities are based on existing capacities reported by IRENA (IRENASTAT) for the specified year +-- expansion_limit,--,float or false,"Artificially limit maximum IRENA capacities to a factor. For example, an ``expansion_limit: 1.1`` means 110% of capacities . If false are chosen, the estimated renewable potentials determine by the workflow are used." +-- technologies,--,list,Switch to select the technologies for which renewable capacities are estimated for overnight foresight. Technologies covered by specified TYNDP renewable carriers need to be removed. +-- technology_mapping,,,Mapping between PyPSA-Eur and powerplantmatching technology names +-- -- Offshore,--,"{onwind}","PyPSA-Eur carrier that is considered for existing onshore wind capacities (IRENA, GEM)." +-- -- Offshore,--,"Any of {offwind-ac, offwind-dc, offwind-float}","PyPSA-Eur carrier that is considered for existing offshore wind technology (IRENA, GEM)." +-- -- PV,--,{solar},"PyPSA-Eur carrier that is considered for existing solar PV capacities (IRENA, GEM)." +autarky,,, +-- enable,bool,true or false,Require each node to be autarkic by removing all lines and links. +-- by_country,bool,true or false,Require each country to be autarkic by removing all cross-border lines and links. ``electricity: autarky`` must be enabled. +transmission_limit,str,"Values like 'vopt', 'v1.25', 'copt', 'c1.25'","Limit on transmission expansion. The first part can be ``v`` (for setting a limit on line volume) or ``c`` (for setting a limit on line cost). The second part can be ``opt`` or a float bigger than one (e.g. 1.25). If ``opt`` is chosen line expansion is optimised according to its capital cost (where the choice ``v`` only considers overhead costs for HVDC transmission lines, while ``c`` uses more accurate costs distinguishing between overhead and underwater sections and including inverter pairs). The setting ``v1.25`` will limit the total volume of line expansion to 25% of currently installed capacities weighted by individual line lengths. The setting ``c1.25`` will allow to build a transmission network that costs no more than 25 % more than the current system." diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 623cd541b2..e02e6590a2 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -13,6 +13,8 @@ Release Notes **Changes** +* Add processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles, replacing default ERA5-based profiles processed with Atlite (https://github.com/open-energy-transition/open-tyndp/pull/53). Initial implementation first addresses profiles for offshore technologies. + * Add TYNDP hydrogen import potentials and corridors from outside of the modelled countries (https://github.com/open-energy-transition/open-tyndp/pull/36). Notably this includes pipelines and shipping imports from North Africa, Ukraine and Norway. Different import potentials are available for each of the planning years which are differentiated by wildcards. * Add the TYNDP electricity demand as an exogenously set demand (https://github.com/open-energy-transition/open-tyndp/pull/14). This requires the default PyPSA-Eur modelling to be explicitly disabled. The TYNDP electricity demand depends on the planning year, necessitating a different approach to the default PyPSA-Eur one. Wildcards are introduced and load is attached in `prepare_sector_network`. diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index ac43d5800b..0797d3aef8 100755 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -393,6 +393,62 @@ rule build_renewable_profiles: "../scripts/build_renewable_profiles.py" +rule clean_pecd_data: + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + input: + offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", + onshore_buses=resources("busmap_base_s_all.csv"), + dir_pecd="data/tyndp_2024_bundle/PECD", + output: + pecd_data_clean=resources("pecd_data_{technology}_{planning_horizons}.csv"), + log: + logs("clean_pecd_data_{technology}_{planning_horizons}.log"), + benchmark: + benchmarks("clean_pecd_data_{technology}_{planning_horizons}") + threads: 4 + resources: + mem_mb=4000, + conda: + "../envs/environment.yaml" + script: + "../scripts/clean_pecd_data.py" + + +def input_data_pecd(w): + return { + f"pecd_data_{pyear}": resources("pecd_data_{technology}_" + str(pyear) + ".csv") + for pyear in set( + config_provider("scenario", "planning_horizons")(w) + ).intersection([2030, 2040]) + # Complete PECD data is only available for the years 2030, 2040 + # TODO: adjust if udpated 2050 data available + } + + +rule build_renewable_profiles_pecd: + params: + planning_horizons=config_provider("scenario", "planning_horizons"), + input: + unpack(input_data_pecd), + output: + profile=resources("profile_pecd_{clusters}_{technology}.nc"), + log: + logs("build_renewable_profile_pecd_{clusters}_{technology}.log"), + benchmark: + benchmarks("build_renewable_profile_pecd_{clusters}_{technology}") + threads: 1 + resources: + mem_mb=4000, + wildcard_constraints: + technology="(?!hydro).*", # Any technology other than hydro + conda: + "../envs/environment.yaml" + script: + "../scripts/build_renewable_profiles_pecd.py" + + rule build_monthly_prices: input: co2_price_raw="data/validation/emission-spot-primary-market-auction-report-2019-data.xls", @@ -736,7 +792,7 @@ def input_profile_tech(w): if tech != "hydro" else f"profile_{tech}.nc" ) - for tech in config_provider("electricity", "renewable_carriers")(w) + for tech in set(config_provider("electricity", "renewable_carriers")(w)) } diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 441a3309a4..1a0a49872d 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1231,6 +1231,19 @@ def input_profile_offwind(w): } +pecd_techs = branch( + config_provider("electricity", "pecd_renewable_profiles", "enable"), + config_provider("electricity", "pecd_renewable_profiles", "technologies"), +) + + +def input_profile_pecd(w): + return { + f"profile_pecd_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") + for tech in pecd_techs(w) + } + + rule build_egs_potentials: params: snapshots=config_provider("snapshots"), @@ -1379,6 +1392,7 @@ rule prepare_sector_network: scaling_factor=config_provider("load", "scaling_factor"), input: unpack(input_profile_offwind), + unpack(input_profile_pecd), unpack(input_heat_source_power), **rules.cluster_gas_network.output, **rules.build_gas_input_locations.output, diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 8a8cf9e1cb..514551171c 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -179,12 +179,27 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_tyndp_bundle" h2_reference_grid="data/tyndp_2024_bundle/Line data/ReferenceGrid_Hydrogen.xlsx", electricity_demand=directory("data/tyndp_2024_bundle/Demand Profiles"), h2_imports="data/tyndp_2024_bundle/Hydrogen/H2 IMPORTS GENERATORS PROPERTIES.xlsx", + offshore_buses="data/tyndp_2024_bundle/Offshore hubs/NODE.xlsx", log: "logs/retrieve_tyndp_bundle.log", retries: 2 script: "../scripts/retrieve_tyndp_bundle.py" + rule retrieve_tyndp_pecd_data: + params: + # TODO Integrate into Zenodo tyndp data bundle + tyndp_bundle="data/tyndp_2024_bundle", + output: + dir=directory("data/tyndp_2024_bundle/PECD"), + log: + "logs/retrieve_tyndp_pecd_data.log", + retries: 2 + script: + "../scripts/retrieve_tyndp_pecd_data.py" + + ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data + rule retrieve_countries_centroids: output: "data/countries_centroids.geojson", diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 2c0296fa3b..473e274520 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -7,6 +7,12 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), + pecd_renewable_profiles=config_provider( + "electricity", "pecd_renewable_profiles" + ), + tyndp_renewable_carriers=config_provider( + "electricity", "tyndp_renewable_carriers" + ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), @@ -60,11 +66,18 @@ rule add_existing_baseyear: def input_profile_tech_brownfield(w): return { f"profile_{tech}": resources("profile_{clusters}_" + tech + ".nc") - for tech in config_provider("electricity", "renewable_carriers")(w) + for tech in (set(config_provider("electricity", "renewable_carriers")(w))) if tech != "hydro" } +def input_profile_tech_brownfield_pecd(w): + return { + f"profile_{tech}": resources("profile_pecd_{clusters}_" + tech + ".nc") + for tech in pecd_techs(w) + } + + rule add_brownfield: params: H2_retrofit=config_provider("sector", "H2_retrofit"), @@ -73,6 +86,12 @@ rule add_brownfield: ), threshold_capacity=config_provider("existing_capacities", "threshold_capacity"), snapshots=config_provider("snapshots"), + pecd_renewable_profiles=config_provider( + "electricity", "pecd_renewable_profiles" + ), + tyndp_renewable_carriers=config_provider( + "electricity", "tyndp_renewable_carriers" + ), drop_leap_day=config_provider("enable", "drop_leap_day"), carriers=config_provider("electricity", "renewable_carriers"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), @@ -82,6 +101,7 @@ rule add_brownfield: ), input: unpack(input_profile_tech_brownfield), + unpack(input_profile_tech_brownfield_pecd), simplify_busmap=resources("busmap_base_s.csv"), cluster_busmap=resources("busmap_base_s_{clusters}.csv"), network=resources( diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index f6530552f6..f8792dc88c 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -5,6 +5,12 @@ rule add_existing_baseyear: params: baseyear=config_provider("scenario", "planning_horizons", 0), sector=config_provider("sector"), + pecd_renewable_profiles=config_provider( + "electricity", "pecd_renewable_profiles" + ), + tyndp_renewable_carriers=config_provider( + "electricity", "tyndp_renewable_carriers" + ), existing_capacities=config_provider("existing_capacities"), carriers=config_provider("electricity", "renewable_carriers"), costs=config_provider("costs"), diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 1e92d6558c..03771db9aa 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Contributors to PyPSA-Eur +# SPDX-FileCopyrightText: Open Energy Transition gGmbH and contributors to PyPSA-Eur # # SPDX-License-Identifier: MIT """ @@ -206,7 +206,9 @@ def disable_grid_expansion_if_limit_hit(n): n.global_constraints.drop(name, inplace=True) -def adjust_renewable_profiles(n, input_profiles, params, year): +def adjust_renewable_profiles( + n, input_profiles, params, year, tyndp_renewable_carriers +): """ Adjusts renewable profiles according to the renewable technology specified, using the latest year below or equal to the selected year. @@ -218,7 +220,12 @@ def adjust_renewable_profiles(n, input_profiles, params, year): pd.Series(dr, index=dr).where(lambda x: x.isin(n.snapshots), pd.NA).ffill() ) - for carrier in params["carriers"]: + # TODO: hotfix remove filter for tyndp_renewable_carriers after tyndp generators are added + if len(tyndp_renewable_carriers) > 0: + logger.info( + f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." + ) + for carrier in set(params["carriers"]) - set(tyndp_renewable_carriers): if carrier == "hydro": continue @@ -349,7 +356,11 @@ def update_dynamic_ptes_capacity( n = pypsa.Network(snakemake.input.network) - adjust_renewable_profiles(n, snakemake.input, snakemake.params, year) + tyndp_renewable_carriers = snakemake.params.tyndp_renewable_carriers + + adjust_renewable_profiles( + n, snakemake.input, snakemake.params, year, tyndp_renewable_carriers + ) add_build_year_to_new_assets(n, year) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 97e325657f..a52a72dd58 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -1215,7 +1215,12 @@ def attach_stores( params.link_length_factor, ) - renewable_carriers = set(params.electricity["renewable_carriers"]) + tyndp_renewable_carriers = params.electricity["tyndp_renewable_carriers"] + if len(tyndp_renewable_carriers) > 0: + logger.info( + f"Skipping TYNDP renewable carriers - they will be attached later with TYNDP data: {', '.join(tyndp_renewable_carriers)}" + ) + renewable_carriers = params.electricity["renewable_carriers"] extendable_carriers = params.electricity["extendable_carriers"] conventional_carriers = params.electricity["conventional_carriers"] conventional_inputs = { @@ -1279,7 +1284,11 @@ def attach_stores( "in rule `add_existing_baseyear` with foresight mode 'myopic'." ) else: - tech_map = estimate_renewable_caps["technology_mapping"] + tech_map = { + key: value + for key, value in estimate_renewable_caps["technology_mapping"].items() + if key in estimate_renewable_caps["technologies"] + } expansion_limit = estimate_renewable_caps["expansion_limit"] year = estimate_renewable_caps["year"] diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index d2526bf3e4..1eae4a215d 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -23,7 +23,10 @@ set_scenario_config, update_config_from_wildcards, ) -from scripts.add_electricity import load_costs, sanitize_carriers +from scripts.add_electricity import ( + load_costs, + sanitize_carriers, +) from scripts.build_energy_totals import cartesian from scripts.definitions.heat_system import HeatSystem from scripts.prepare_sector_network import cluster_heat_buses, define_spatial @@ -69,6 +72,7 @@ def add_existing_renewables( df_agg: pd.DataFrame, countries: list[str], renewable_carriers: list[str], + tyndp_renewable_carriers: list[str], ) -> None: """ Add existing renewable capacities to conventional power plant data. @@ -85,6 +89,8 @@ def add_existing_renewables( List of country codes to consider renewable_carriers: list List of renewable carriers in the network + tyndp_renewable_carriers: list + List of renewable technologies from TYNDP Returns ------- @@ -92,6 +98,12 @@ def add_existing_renewables( Modifies df_agg in-place """ tech_map = {"solar": "PV", "onwind": "Onshore", "offwind-ac": "Offshore"} + # TODO: remove when TYNDP renewable generators are added + if len(tyndp_renewable_carriers) > 0: + logger.info( + f"Hotfix until TYNDP renewable carriers are added. Skipping renewable carriers '{', '.join(tyndp_renewable_carriers)}'." + ) + renewable_carriers = set(renewable_carriers) - set(tyndp_renewable_carriers) irena = pm.data.IRENASTAT().powerplant.convert_country_to_alpha2() irena = irena.query("Country in @countries") @@ -155,6 +167,7 @@ def add_power_capacities_installed_before_baseyear( capacity_threshold: float, lifetime_values: dict[str, float], renewable_carriers: list[str], + tyndp_renewable_carriers: list[str], ) -> None: """ Add power generation capacities installed before base year. @@ -179,6 +192,8 @@ def add_power_capacities_installed_before_baseyear( Default values for missing data renewable_carriers: list List of renewable carriers in the network + tyndp_renewable_carriers: list + List of renewable technologies from TYNDP """ logger.debug(f"Adding power capacities installed before {baseyear}") @@ -232,6 +247,7 @@ def add_power_capacities_installed_before_baseyear( n=n, countries=countries, renewable_carriers=renewable_carriers, + tyndp_renewable_carriers=tyndp_renewable_carriers, ) # drop assets which are already phased out / decommissioned phased_out = df_agg[df_agg["DateOut"] < baseyear].index @@ -744,6 +760,7 @@ def add_heating_capacities_installed_before_baseyear( options = snakemake.params.sector renewable_carriers = snakemake.params.carriers + tyndp_renewable_carriers = snakemake.params.tyndp_renewable_carriers baseyear = snakemake.params.baseyear @@ -772,6 +789,7 @@ def add_heating_capacities_installed_before_baseyear( capacity_threshold=snakemake.params.existing_capacities["threshold_capacity"], lifetime_values=snakemake.params.costs["fill_values"], renewable_carriers=renewable_carriers, + tyndp_renewable_carriers=tyndp_renewable_carriers, ) if options["heating"]: diff --git a/scripts/build_renewable_profiles_pecd.py b/scripts/build_renewable_profiles_pecd.py new file mode 100644 index 0000000000..67646349b6 --- /dev/null +++ b/scripts/build_renewable_profiles_pecd.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: MIT +""" +Create renewable profiles for each region from PECD and TYNDP datasets, building on PECD climate data. The available +generation time series are read for each node across all renewable technologies, including onshore wind, AC-connected offshore wind, +DC-connected offshore wind, and solar PV generators. + +.. note:: Hydroelectric profiles will be built in script :mod:`build_hydro_profiles_PECD`. Not yet implemented. + +Outputs +------- + +- ``resources/profile_pecd_{clusters}_{technology}.nc`` with the following structure + + =================== ==================== ========================================================= + Field Dimensions Description + =================== ==================== ========================================================= + profile year, bus, bin, time the per unit hourly availability factors for each bus + =================== ==================== ========================================================= +""" + +import logging + +import numpy as np +import pandas as pd +import xarray as xr + +from scripts._helpers import ( + configure_logging, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake( + "build_renewable_profiles_pecd", + clusters="all", + technology="Wind_Offshore", + ) + configure_logging(snakemake) + set_scenario_config(snakemake) + + technology = snakemake.wildcards.technology + pyears = snakemake.params.planning_horizons + + profiles = [] + + for year in pyears: + logger.info( + f"Extract PECD capacity factor time series for year {year} for technology {technology}..." + ) + year_i = year + if int(year) not in [2030, 2040, 2050]: + year = np.clip(10 * (year // 10), 2030, 2050) + logger.warning( + f"TYNDP PECD data unavailable for planning horizon. Falling back to previous available year {year}." + ) + if year == 2050: + logger.warning( + "PECD input data for 2050 is incomplete. Falling back to 2040 PECD data." + ) + year = 2040 + + profile = ( + pd.read_csv( + snakemake.input[f"pecd_data_{year}"], parse_dates=True, index_col=0 + ) + .rename_axis("time") + .reset_index() + .melt(id_vars=["time"], var_name="bus", value_name="profile") + .assign(bin=0, year=year_i) + .set_index(["time", "bus", "bin", "year"]) + .to_xarray() + ) + + profiles.append(profile) + + ds = xr.merge(profiles) + ds.to_netcdf(snakemake.output.profile) diff --git a/scripts/build_tyndp_h2_network.py b/scripts/build_tyndp_h2_network.py index 29d57dbcfe..c0c83e2090 100644 --- a/scripts/build_tyndp_h2_network.py +++ b/scripts/build_tyndp_h2_network.py @@ -10,6 +10,7 @@ import logging +import numpy as np import pandas as pd from _helpers import ( configure_logging, @@ -60,7 +61,7 @@ def load_h2_interzonal_connections(fn, scenario="GA", pyear=2030): "Planning horizon doesn't match available TYNDP data. " "Falling back to closest available year between 2030 and 2050." ) - pyear = min(max(2030, 5 * round(pyear / 5)), 2050) + pyear = np.clip(5 * (pyear // 5), 2030, 2050) scenario_dict = { "GA": "Global Ambition", "DE": "Distributed Energy", diff --git a/scripts/clean_pecd_data.py b/scripts/clean_pecd_data.py new file mode 100644 index 0000000000..c582b08b1d --- /dev/null +++ b/scripts/clean_pecd_data.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: MIT +""" +Loads and cleans the available PECD capacity factor generation time series based on PECD weather data. +The script is executed for a given technology, and planning horizon. Technologies can be one of: + + * CSP_noStorage, + * CSP_withStorage, + * LFSolarPV, + * LFSolarPVRooftop, + * LFSolarPVUtility, + * Wind_Offshore, + * Wind_Onshore. + +Outputs +------- +Cleaned csv file with capacity factor generation time series and regions as columns. +""" + +import logging +import multiprocessing as mp +import os +from functools import partial +from pathlib import Path + +import pandas as pd +from tqdm import tqdm + +from scripts._helpers import ( + configure_logging, + get_snapshots, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + + +def read_pecd_file( + node: str, + dir_pecd: str, + cyear: str, + pyear: str, + technology: str, + sns: pd.DatetimeIndex, +): + fn = Path(dir_pecd, pyear, f"PECD_{technology}_{pyear}_{node}_edition 2023.2.csv") + + if not os.path.isfile(fn): + return None + + pecd_bus = pd.read_csv( + fn, + skiprows=10, # first ten rows contain only file metadata + usecols=lambda name: name == "Date" + or name == "Hour" + or name == str(cyear) + or name == str(float(cyear)), + ).rename(columns={str(float(cyear)): str(cyear)}) + + datetime_str = f"{cyear}." + pecd_bus["Date"].str.cat( + (pecd_bus["Hour"] - 1).astype(str), sep=" " + ) + cf_pecd = ( + pecd_bus.set_index(pd.to_datetime(datetime_str, format="%Y.%d.%m. %H")) + .drop(columns=["Date", "Hour"]) + .rename(columns={str(cyear): node}) + .loc[sns] # filter for snapshots only + ) + + return cf_pecd + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake( + "clean_pecd_data", + clusters="all", + technology="Wind_Offshore", + planning_horizons=2030, + ) + configure_logging(snakemake) + set_scenario_config(snakemake) + + # Climate year from snapshots + sns = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day) + cyear = sns[0].year + if int(cyear) < 1982 or int(cyear) > 2019: + # TODO: Note that because of this fallback, the snapshots of the profiles will not always match with the model snapshots + logger.warning( + "Snapshot year doesn't match available TYNDP data. Falling back to 2009." + ) + cyear = 2009 + + # Planning year + pyear = str(snakemake.wildcards.planning_horizons) + + # TODO: find solution for solar profiles being differentiated between Utility and Rooftop for Italy + # Technology as in PECD terminology + pecd_tech = snakemake.wildcards.technology + + offshore_buses = pd.read_excel(snakemake.input.offshore_buses, index_col=0) + onshore_buses = pd.read_csv(snakemake.input.onshore_buses, index_col=0) + + nodes = ( + offshore_buses.index if pecd_tech == "Wind_Offshore" else onshore_buses.index + ) + dir_pecd = snakemake.input.dir_pecd + + # Load and prep electricity demand + tqdm_kwargs = { + "ascii": False, + "unit": " nodes", + "total": len(nodes), + "desc": "Loading PECD capacity factor data", + } + + func = partial( + read_pecd_file, + dir_pecd=dir_pecd, + cyear=cyear, + pyear=pyear, + technology=pecd_tech, + sns=sns, + ) + + with mp.Pool(processes=snakemake.threads) as pool: + demand = list(tqdm(pool.imap(func, nodes), **tqdm_kwargs)) + + pecd_df = ( + pd.concat(demand, axis=1) + .reindex( + nodes, axis=1, fill_value=0.0 + ) # include missing node data with empty columns + .rename( + columns=lambda x: x.replace("UK", "GB") + ) # replace UK with GB for naming convention + ) + + pecd_df.to_csv(snakemake.output.pecd_data_clean) diff --git a/scripts/retrieve_tyndp_pecd_data.py b/scripts/retrieve_tyndp_pecd_data.py new file mode 100644 index 0000000000..8dc9ffdb1b --- /dev/null +++ b/scripts/retrieve_tyndp_pecd_data.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Open Energy Transition gGmbH +# +# SPDX-License-Identifier: MIT +""" +The TYNDP PECD data contains input data for the 2024 TYNDP scenario building process. + +This rule downloads the TYNDP PECD v3.1 data from Google Drive and extracts it in the ``data/tyndp_2024_bundle`` +subdirectory, such that all files of the TYNDP bundle are stored in it. + +**Outputs** + +- ``data/tyndp_2024_bundle/PECD``: PECD input data for TYNDP 2024 scenario building + +""" + +import logging +import os +import zipfile +from pathlib import Path + +from _helpers import configure_logging, progress_retrieve, set_scenario_config + +logger = logging.getLogger(__name__) + +# Define the base URL +# TODO: retrieve_tyndp_pecd_data needs to be deprecated once PECD data is added to the TYNDP data bundle +url = "https://storage.googleapis.com/open-tyndp-data-store/PECD.zip" + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_tyndp_pecd_data") + rootpath = ".." + else: + rootpath = "." + + configure_logging(snakemake) + set_scenario_config(snakemake) + disable_progress = snakemake.config["run"].get("disable_progressbar", False) + + to_fn = snakemake.output.dir + tyndp_bundle_fn = Path(rootpath, snakemake.params["tyndp_bundle"]) + to_fn_zp = to_fn + ".zip" + + # download .zip file + logger.info(f"Downloading TYNDP PECD data from '{url}'.") + progress_retrieve(url, to_fn_zp, disable=disable_progress) + + # extract + logger.info("Extracting TYNDP PECD data.") + with zipfile.ZipFile(to_fn_zp, "r") as zip_ref: + zip_ref.extractall(tyndp_bundle_fn) + + # remove .zip file + os.remove(to_fn_zp) + + logger.info(f"TYNDP PECD data available in '{to_fn}'.")