feat: introduce PEMMDB data and tyndp trajectories

.github/pull_request_template.md

@@ -25,5 +25,6 @@ Closes # (if applicable).
 - [ ] Changes in configuration options are added in `config/test/*.yaml`.
 - [ ] OET license identifier is added to all edited or newly created code files.
 - [ ] Sources of newly added data are documented in `doc/data_sources.rst`.
+- [ ] New rules are documented in the appropriate `doc/*.rst` files.
 - [ ] A release note `doc/release_notes.rst` is added.
 - [ ] Major features are documented with up-to-date information in `README` and `doc/index.rst`.

Makefile

@@ -79,29 +79,29 @@ install-pinned-macos: _conda_check
 # Run default tests
 test:
 	set -e
+	snakemake -call --configfile config/test/config.tyndp.yaml
+	snakemake -call --configfile config/config.tyndp.yaml -n
 	snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml
 	snakemake -call --configfile config/test/config.overnight.yaml
 	snakemake -call --configfile config/test/config.myopic.yaml
 	snakemake -call make_summary_perfect --configfile config/test/config.perfect.yaml
 	snakemake -call resources/test-elec-clusters/networks/base_s_adm.nc --configfile config/test/config.clusters.yaml
 	snakemake -call --configfile config/test/config.scenarios.yaml -n
-	snakemake -call --configfile config/test/config.tyndp.yaml
-	snakemake -call --configfile config/config.tyndp.yaml -n
 	echo "All tests completed successfully."
 
 unit-test:
 	pytest test
 
 # Cleans all output files from tests
 clean-tests:
+	snakemake -call --configfile config/test/config.tyndp.yaml --delete-all-output
+	snakemake -call --configfile config/config.tyndp.yaml -n --delete-all-output
 	snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --delete-all-output
 	snakemake -call --configfile config/test/config.overnight.yaml --delete-all-output
 	snakemake -call --configfile config/test/config.myopic.yaml --delete-all-output
 	snakemake -call make_summary_perfect --configfile config/test/config.perfect.yaml --delete-all-output
 	snakemake -call resources/test-elec-clusters/networks/base_s_adm.nc --configfile config/test/config.clusters.yaml --delete-all-output
 	snakemake -call --configfile config/test/config.scenarios.yaml -n --delete-all-output
-	snakemake -call --configfile config/test/config.tyndp.yaml --delete-all-output
-	snakemake -call --configfile config/config.tyndp.yaml -n --delete-all-output
 
 # Removes all created files except for large cutout files (similar to fresh clone)
 reset:

README.md

@@ -45,6 +45,7 @@ The back-casting of the 2024 TYNDP cycle involves developing new features based
 - Added processing and preparation of TYNDP 2024 PECD v3.1 renewable profiles for offshore wind farms, replacing default ERA5-based profiles processed with Atlite (see PR [#53](https://github.com/open-energy-transition/open-tyndp/pull/53)).
 - Introduced TYNDP offshore wind hubs topology with both electric and hydrogen infrastructure, offshore electrolysers, and detailed wind farm characteristics (see PR [#54](https://github.com/open-energy-transition/open-tyndp/pull/54)).
 - Introduce a benchmarking framework that assesses Open-TYNDP quality against published TYNDP 2024 data (see PR [#73](https://github.com/open-energy-transition/open-tyndp/pull/73)) and Visualisation Platform data (see PR [#117](https://github.com/open-energy-transition/open-tyndp/pull/117)).
+- Added processing and preparation of PEMMDB v2.4 capacity, must-run, and availability data, along with expansion trajectories for conventional and renewable power generation, electrolysers, batteries, and DSR (see PR [#97](https://github.com/open-energy-transition/open-tyndp/pull/97)).
 
 |       Feature        | TYNDP 2024 topology | Open-TYNDP topology |
 |:--------------------:|:-------------------:|:-------------------:|
@@ -60,16 +61,16 @@ While multiple TYNDP features are already introduced to the Open-TYNDP model, th
 |:------------------------------------------------------:|:---------------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------------------------:|:------:|
 |       **Visualizations and workflow automation**       |                 Automated workflow                  |                                                                                                                                            |   ✅    |
 |                                                        |               TYNDP plotting routines               |                                                                                                                                            |   ⌛    |
-|           **Automated tests and benchmarks**           |           Automated benchmarking routine            | [#73](https://github.com/open-energy-transition/open-tyndp/pull/73), [#117](https://github.com/open-energy-transition/open-tyndp/pull/117) |   ✅   |
+|           **Automated tests and benchmarks**           |           Automated benchmarking routine            | [#73](https://github.com/open-energy-transition/open-tyndp/pull/73), [#117](https://github.com/open-energy-transition/open-tyndp/pull/117) |   ✅    |
 |              **TYNDP modelling features**              |           Perfect foresight optimization            |                                                                                                                                            |   ⌛    |
 |                                                        |            Security of Supply (SoS) loop            |                                                                                                                                            |   ⌛    |
 | **Existing infrastructure and associated parameters**  |             Electricity reference grid              |                                    [#18](https://github.com/open-energy-transition/open-tyndp/pull/18)                                     |   ✅    |
 |                                                        |               Hydrogen reference grid               |  [#17](https://github.com/open-energy-transition/open-tyndp/pull/17), [#36](https://github.com/open-energy-transition/open-tyndp/pull/36)  |   ✅    |
 |                                                        |                    Offshore grid                    |                                    [#54](https://github.com/open-energy-transition/open-tyndp/pull/54)                                     |   ✅    |
 |                                                        |                      PECD data                      |  [#53](https://github.com/open-energy-transition/open-tyndp/pull/53), [#71](https://github.com/open-energy-transition/open-tyndp/pull/71)  |   ✅    |
 |                                                        |                    Hydro inflows                    |                                    [#77](https://github.com/open-energy-transition/open-tyndp/pull/77)                                     |   ✅    |
-|                                                        |            PEMMDB capacities & must-runs            |                                    [#97](https://github.com/open-energy-transition/open-tyndp/pull/97)                                     |   🔨   |
-|                                                        |         Investment candidates trajectories          |                                    [#97](https://github.com/open-energy-transition/open-tyndp/pull/97)                                     |   🔨   |
+|                                                        |      PEMMDB capacities & must-runs processing       |                                    [#97](https://github.com/open-energy-transition/open-tyndp/pull/97)                                     |   ✅    |
+|                                                        |    Investment candidates trajectories processing    |                                    [#97](https://github.com/open-energy-transition/open-tyndp/pull/97)                                     |   ✅    |
 |                    **TYNDP demand**                    |                     Electricity                     |                                    [#14](https://github.com/open-energy-transition/open-tyndp/pull/14)                                     |   ✅    |
 |                                                        |                      Hydrogen                       |                                                                                                                                            |   ⌛    |
 |                                                        |                       Methane                       |                                                                                                                                            |   ⌛    |

REUSE.toml

@@ -21,6 +21,7 @@ SPDX-License-Identifier = "CC-BY-4.0"
 [[annotations]]
 path = [
     "data/tyndp_electricity_loss_factors.csv",
+    "data/tyndp_technology_map.csv",
     "doc/configtables/benchmarking.csv",
     "doc/img/tyndp/*",
 ]

config/config.default.yaml

@@ -140,8 +140,10 @@ electricity:
   everywhere_powerplants: []
 
   conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass]
+  tyndp_conventional_carriers: []
   renewable_carriers: [solar, solar-hsat, onwind, offwind-ac, offwind-dc, offwind-float, hydro]
   tyndp_renewable_carriers: []
+  tyndp_stores: []
 
   pecd_renewable_profiles:
     enable: false
@@ -159,15 +161,10 @@ electricity:
     - 2040
     - 2050
     technologies:
-      Wind_Offshore:
-      - offwind-ac-fb-r
-      - offwind-ac-fl-r
-      - offwind-dc-fb-r
-      - offwind-dc-fl-r
-      - offwind-dc-fb-oh
-      - offwind-dc-fl-oh
-      - offwind-h2-fb-oh
-      - offwind-h2-fl-oh
+    - Wind_Offshore
+    - LFSolarPVRooftop
+    - LFSolarPVUtility
+    - Wind_Onshore
 
   pemmdb_hydro_profiles:
     enable: false
@@ -182,6 +179,30 @@ electricity:
     - PS Open
     - PS Closed
 
+  pemmdb_capacities:
+    enable: false
+    nprocesses: 4
+    available_years:
+    - 2030
+    - 2040
+    - 2050
+    technologies:
+    - Solar
+    - Wind
+    - Hydro
+    - Other_RES
+    - Gas
+    - Nuclear
+    - Hard_coal
+    - Lignite
+    - Light_oil
+    - Heavy_oil
+    - Oil_shale
+    - Other_Non-RES
+    - Hydrogen
+    - Electrolyser
+    - Battery
+
   estimate_renewable_capacities:
     enable: true
     from_gem: true

config/config.tyndp.yaml

@@ -42,8 +42,37 @@ electricity:
   base_network: tyndp
 
   # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
+  tyndp_conventional_carriers:
+  - gas
+  - coal
+  - lignite
+  - nuclear
+  - oil-light
+  - oil-heavy
+  - oil-shale
   renewable_carriers: [hydro, solar, solar-hsat, onwind]
-  tyndp_renewable_carriers: [hydro, onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
+  tyndp_renewable_carriers:
+  - other-res
+  - hydro-ror
+  - hydro-reservoir
+  - hydro-pondage
+  - hydro-phs
+  - hydro-phs-pure
+  - onwind
+  - solar-pv-utility
+  - solar-pv-rooftop
+  - offwind-ac-fb-r
+  - offwind-ac-fl-r
+  - offwind-dc-fb-r
+  - offwind-dc-fl-r
+  - offwind-dc-fb-oh
+  - offwind-dc-fl-oh
+  - offwind-h2-fb-oh
+  - offwind-h2-fl-oh
+  tyndp_stores:
+  - battery
+  - h2_cavern
+  - h2_tank
 
   pecd_renewable_profiles:
     enable: true
@@ -62,21 +91,10 @@ electricity:
     - 2030
     - 2040
     technologies:
-      Wind_Offshore:
-      - offwind-ac-fb-r
-      - offwind-ac-fl-r
-      - offwind-dc-fb-r
-      - offwind-dc-fl-r
-      - offwind-dc-fb-oh
-      - offwind-dc-fl-oh
-      - offwind-h2-fb-oh
-      - offwind-h2-fl-oh
-      LFSolarPVRooftop:
-      - solar-pv-rooftop
-      LFSolarPVUtility:
-      - solar-pv-utility
-      Wind_Onshore:
-      - onwind
+    - Wind_Offshore
+    - LFSolarPVRooftop
+    - LFSolarPVUtility
+    - Wind_Onshore
 
   pemmdb_hydro_profiles:
     enable: true
@@ -85,11 +103,35 @@ electricity:
     - 2040
     - 2050
     technologies:
-    - Run of River
+    - Run_of_River
     - Pondage
     - Reservoir
-    - PS Open
-    - PS Closed
+    - PS_Open
+    - PS_Closed
+
+  pemmdb_capacities:
+    enable: true
+    nprocesses: 4
+    available_years:
+    - 2030
+    - 2040
+    - 2050
+    technologies:
+    - Solar
+    - Wind
+    - Hydro
+    - Other_RES
+    - Gas
+    - Nuclear
+    - Hard_coal
+    - Lignite
+    - Light_oil
+    - Heavy_oil
+    - Oil_shale
+    - Other_Non-RES
+    - Hydrogen
+    - Electrolyser
+    - Battery
 
   estimate_renewable_capacities:
     # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation

config/test/config.tyndp.yaml

@@ -49,8 +49,37 @@ electricity:
     Link: [H2 pipeline]
 
   # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
+  tyndp_conventional_carriers:
+  - gas
+  - coal
+  - lignite
+  - nuclear
+  - oil-light
+  - oil-heavy
+  - oil-shale
   renewable_carriers: [solar, solar-hsat, onwind]
-  tyndp_renewable_carriers: [hydro, onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
+  tyndp_renewable_carriers:
+  - other-res
+  - hydro-ror
+  - hydro-reservoir
+  - hydro-pondage
+  - hydro-phs
+  - hydro-phs-pure
+  - onwind
+  - solar-pv-utility
+  - solar-pv-rooftop
+  - offwind-ac-fb-r
+  - offwind-ac-fl-r
+  - offwind-dc-fb-r
+  - offwind-dc-fl-r
+  - offwind-dc-fb-oh
+  - offwind-dc-fl-oh
+  - offwind-h2-fb-oh
+  - offwind-h2-fl-oh
+  tyndp_stores:
+  - battery
+  - h2_cavern
+  - h2_tank
 
   pecd_renewable_profiles:
     enable: true
@@ -69,21 +98,10 @@ electricity:
     - 2030
     - 2040
     technologies:
-      Wind_Offshore:
-      - offwind-ac-fb-r
-      - offwind-ac-fl-r
-      - offwind-dc-fb-r
-      - offwind-dc-fl-r
-      - offwind-dc-fb-oh
-      - offwind-dc-fl-oh
-      - offwind-h2-fb-oh
-      - offwind-h2-fl-oh
-      LFSolarPVRooftop:
-      - solar-pv-rooftop
-      LFSolarPVUtility:
-      - solar-pv-utility
-      Wind_Onshore:
-      - onwind
+    - Wind_Offshore
+    - LFSolarPVRooftop
+    - LFSolarPVUtility
+    - Wind_Onshore
 
   pemmdb_hydro_profiles:
     enable: true
@@ -92,11 +110,35 @@ electricity:
     - 2040
     - 2050
     technologies:
-    - Run of River
+    - Run_of_River
     - Pondage
     - Reservoir
-    - PS Open
-    - PS Closed
+    - PS_Open
+    - PS_Closed
+
+  pemmdb_capacities:
+    enable: true
+    nprocesses: 4
+    available_years:
+    - 2030
+    - 2040
+    - 2050
+    technologies:
+    - Solar
+    - Wind
+    - Hydro
+    - Other_RES
+    - Gas
+    - Nuclear
+    - Hard_coal
+    - Lignite
+    - Light_oil
+    - Heavy_oil
+    - Oil_shale
+    - Other_Non-RES
+    - Hydrogen
+    - Electrolyser
+    - Battery
 
   estimate_renewable_capacities:
     # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation