feat: Integrate hydro inflows from PEMMDB

config/config.default.yaml

@@ -161,6 +161,19 @@ electricity:
       - offwind-h2-fb-oh
       - offwind-h2-fl-oh
 
+  tyndp_hydro_profiles:
+    enable: false
+    technologies:
+    - Run of River
+    - Pondage
+    - Reservoir
+    - PS Open
+    - PS Closed
+    available_years:
+    - 2030
+    - 2040
+    - 2050
+
   estimate_renewable_capacities:
     enable: true
     from_gem: true

config/config.tyndp.yaml

@@ -38,8 +38,8 @@ electricity:
   base_network: tyndp
 
   # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
-  renewable_carriers: [onwind, solar, solar-hsat, hydro]
-  tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
+  renewable_carriers: [solar, solar-hsat, onwind]
+  tyndp_renewable_carriers: [hydro, onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
 
   pecd_renewable_profiles:
     enable: true
@@ -66,6 +66,19 @@ electricity:
       Wind_Onshore:
       - onwind
 
+  tyndp_hydro_profiles:
+    enable: true
+    technologies:
+    - Run of River
+    - Pondage
+    - Reservoir
+    - PS Open
+    - PS Closed
+    available_years:
+    - 2030
+    - 2040
+    - 2050
+
   estimate_renewable_capacities:
     # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation
     technologies:

config/test/config.tyndp.yaml

@@ -50,8 +50,8 @@ electricity:
     Link: [H2 pipeline]
 
   # TODO: remove redundant onwind and solar from renewable_carriers list once technologies are included as tyndp_renewable_carriers from PEMMDB data
-  renewable_carriers: [onwind, solar, solar-hsat]
-  tyndp_renewable_carriers: [onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
+  renewable_carriers: [solar, solar-hsat, onwind]
+  tyndp_renewable_carriers: [hydro, onwind, solar-pv-utility, solar-pv-rooftop, offwind-ac-fb-r, offwind-ac-fl-r, offwind-dc-fb-r, offwind-dc-fl-r, offwind-dc-fb-oh, offwind-dc-fl-oh, offwind-h2-fb-oh, offwind-h2-fl-oh]
 
   pecd_renewable_profiles:
     enable: true
@@ -78,6 +78,19 @@ electricity:
       Wind_Onshore:
       - onwind
 
+  tyndp_hydro_profiles:
+    enable: true
+    technologies:
+    - Run of River
+    - Pondage
+    - Reservoir
+    - PS Open
+    - PS Closed
+    available_years:
+    - 2030
+    - 2040
+    - 2050
+
   estimate_renewable_capacities:
     # NOTE: technologies that are covered by TYNDP renewable carriers need to be removed from estimation
     technologies:

doc/configtables/electricity.csv

@@ -38,6 +38,10 @@ pecd_renewable_profiles,,,
 -- technologies,,,
 -- -- {pecd_tech},--,str,The PECD tech whose PECD profile is used. These PECD tech and their profiles are mapped to ``tyndp_renewable_carriers``.
 -- -- -- {tyndp_renewable_carriers},--,str,The `tyndp_renewable_carriers` for which the PECD profiles must be used. All `tyndp_renewable_carriers` must be mapped to a PECD profile. Non-TYNDP renewable carriers use the default renewable profiles.
+tyndp_hydro_profiles,,,
+-- enable,,bool,Activate PEMMDB hyddro inflow profiles from 2024 TYNDP instead of default hydro profiles.
+-- available_years,--,list,"List of years for which PEMMDB hydro inflows data is available."
+-- technologies,--,list,The hydro technology for which PEMMDB hydro inflows data shall be prepared. Technologies can be any of {Run of River, Pondage, Reservoir, PS Open, PS Closed}.
 estimate_renewable_capacities,,,
 -- enable,,bool,Activate routine to estimate renewable capacities in rule :mod:`add_electricity`. This option should not be used in combination with pathway planning ``foresight: myopic`` or ``foresight: perfect`` as renewable capacities are added differently in :mod:`add_existing_baseyear`.
 -- from_gem,--,bool,Add renewable capacities from `Global Energy Monitor's Global Solar Power Tracker <https://globalenergymonitor.org/projects/global-solar-power-tracker/>`_ and `Global Energy Monitor's Global Wind Power Tracker <https://globalenergymonitor.org/projects/global-wind-power-tracker/>`_.

doc/release_notes.rst

@@ -14,6 +14,8 @@ Upcoming Open-TYNDP Release
 
 * Add complete compatibility for processing and preparation of PECD v3.1 renewable profiles (Solar PV rooftop, Solar PV utility, Onshore Wind, Offshore Wind, Solar CSP) (https://github.com/open-energy-transition/open-tyndp/pull/71). These profiles are used for the TYNDP 2024 and replace the default ERA5- and SARAH3-based profiles processed with Atlite. This implementation serves to facilitate a sub-workflow for creation of the renewable profiles, but does not yet attach them to any technologies.
 
+* Introduce processing of PEMMDB hydro inflows data for different hydro technologies (Run of River, Pondage, Reservoir, PS Open, PS Closed) from 2024 TYNDP into hydro inflow profiles (https://github.com/open-energy-transition/open-tyndp/pull/77). This implementation serves to facilitate a sub-workflow for creation of the hydro inflow profiles, but does not yet attach them to any hydro technologies.
+
 
 Upcoming PyPSA-Eur Release
 ================

doc/retrieve.rst

@@ -172,4 +172,9 @@ None.
 Rule ``retrieve_tyndp_pecd_data``
 ====================================
 
-.. automodule:: retrieve_tyndp_pecd_data
+.. automodule:: retrieve_additional_tyndp_data
+
+Rule ``retrieve_tyndp_hydro_inflows``
+====================================
+
+.. automodule:: retrieve_additional_tyndp_data

rules/build_electricity.smk

@@ -513,6 +513,79 @@ rule build_hydro_profile:
         "../scripts/build_hydro_profile.py"
 
 
+rule clean_tyndp_hydro_inflows:
+    params:
+        snapshots=config_provider("snapshots"),
+        drop_leap_day=config_provider("enable", "drop_leap_day"),
+    input:
+        hydro_inflows_dir="data/tyndp_2024_bundle/Hydro Inflows",
+        onshore_buses=resources("busmap_base_s_all.csv"),
+    output:
+        hydro_inflows_tyndp=resources(
+            "hydro_inflows_tyndp_{tech}_{planning_horizons}.csv"
+        ),
+    log:
+        logs("clean_tyndp_hydro_inflows_{tech}_{planning_horizons}.log"),
+    threads: 4
+    benchmark:
+        benchmarks("clean_tyndp_hydro_inflows_{tech}_{planning_horizons}")
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/clean_tyndp_hydro_inflows.py"
+
+
+def input_data_hydro_tyndp(w):
+    return {
+        f"hydro_inflow_tyndp_{tech}_{pyear}": resources(
+            f"hydro_inflows_tyndp_{tech}_{str(pyear)}.csv"
+        )
+        for pyear in set(
+            [
+                safe_pyear(
+                    year,
+                    config_provider(
+                        "electricity", "tyndp_hydro_profiles", "available_years"
+                    )(w),
+                    "PEMMDB hydro",
+                    verbose=False,
+                )
+                for year in config_provider("scenario", "planning_horizons")(w)
+            ]
+        )
+        for tech in config_provider(
+            "electricity", "tyndp_hydro_profiles", "technologies"
+        )(w)
+    }
+
+
+rule build_tyndp_hydro_profile:
+    params:
+        snapshots=config_provider("snapshots"),
+        drop_leap_day=config_provider("enable", "drop_leap_day"),
+        planning_horizons=config_provider("scenario", "planning_horizons"),
+        available_years=config_provider(
+            "electricity", "tyndp_hydro_profiles", "available_years"
+        ),
+        technologies=config_provider(
+            "electricity", "tyndp_hydro_profiles", "technologies"
+        ),
+    input:
+        unpack(input_data_hydro_tyndp),
+    output:
+        profile=resources("profile_hydro_tyndp.nc"),
+    log:
+        logs("build_tyndp_hydro_profile.log"),
+    benchmark:
+        benchmarks("build_tyndp_hydro_profile")
+    resources:
+        mem_mb=5000,
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_tyndp_hydro_profile.py"
+
+
 rule build_line_rating:
     params:
         snapshots=config_provider("snapshots"),

rules/build_sector.smk

@@ -1573,6 +1573,11 @@ rule prepare_sector_network:
             if config_provider("sector", "h2_topology_tyndp")(w)
             else []
         ),
+        profile_hydro_tyndp=branch(
+            config_provider("electricity", "tyndp_hydro_profiles", "enable"),
+            resources("profile_hydro_tyndp.nc"),
+            [],
+        ),
     output:
         resources(
             "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc"

rules/retrieve.smk

@@ -194,15 +194,30 @@ if config["enable"]["retrieve"]:
         params:
             # TODO Integrate into Zenodo tyndp data bundle
             tyndp_bundle="data/tyndp_2024_bundle",
+            url="https://storage.googleapis.com/open-tyndp-data-store/PECD.zip",
         output:
             dir=directory("data/tyndp_2024_bundle/PECD"),
         log:
             "logs/retrieve_tyndp_pecd_data.log",
         retries: 2
         script:
-            "../scripts/retrieve_tyndp_pecd_data.py"
+            "../scripts/retrieve_additional_tyndp_data.py"
+
+    rule retrieve_tyndp_hydro_inflows:
+        params:
+            # TODO Integrate into Zenodo tyndp data bundle
+            tyndp_bundle="data/tyndp_2024_bundle",
+            url="https://storage.googleapis.com/open-tyndp-data-store/Hydro_Inflows.zip",
+        output:
+            dir=directory("data/tyndp_2024_bundle/Hydro Inflows"),
+        log:
+            "logs/retrieve_tyndp_hydro_inflows.log",
+        retries: 2
+        script:
+            "../scripts/retrieve_additional_tyndp_data.py"
 
     ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_pecd_data > clean_pecd_data
+    ruleorder: retrieve_tyndp_bundle > retrieve_tyndp_hydro_inflows > clean_tyndp_hydro_inflows
 
     rule retrieve_countries_centroids:
         output:

scripts/build_tyndp_hydro_profile.py

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: Open Energy Transition gGmbH
+#
+# SPDX-License-Identifier: MIT
+"""
+Build hydroelectric inflow time-series for each country based on TYNDP hydro inflow data.
+
+Outputs
+-------
+
+- ``resources/profile_hydro_tyndp.nc``:
+
+    ===================  ================  =========================================================
+    Field                Dimensions        Description
+    ===================  ================  =========================================================
+    inflow               bus, time,         Inflow to the state of charge (in MW),
+                         year, hydro_tech   e.g. due to river inflow in hydro reservoir.
+    ===================  ================  =========================================================
+"""
+
+import logging
+
+import pandas as pd
+import xarray as xr
+from tqdm.contrib.itertools import product
+
+from scripts._helpers import (
+    configure_logging,
+    get_snapshots,
+    safe_pyear,
+    set_scenario_config,
+)
+
+logger = logging.getLogger(__name__)
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from scripts._helpers import mock_snakemake
+
+        snakemake = mock_snakemake("build_tyndp_hydro_profile")
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
+
+    time = get_snapshots(snakemake.params.snapshots, snakemake.params.drop_leap_day)
+
+    years_in_time = pd.DatetimeIndex(time).year.unique()
+
+    technologies = snakemake.params.technologies
+    pyears = snakemake.params.planning_horizons
+
+    inflows = []
+
+    for year, technology in product(pyears, technologies):
+        logger.info(
+            f"Extracting hydro inflows for year {year} for technology {technology}..."
+        )
+        year_i = year
+        # falling back to latest available pyear if not in list of available years
+        year = safe_pyear(
+            int(year),
+            available_years=snakemake.params.available_years,
+            source="PEMMDB hydro inflow",
+        )
+
+        inflow = (
+            pd.read_csv(
+                snakemake.input[f"hydro_inflow_tyndp_{technology}_{year}"],
+                parse_dates=True,
+                index_col=0,
+            )
+            .rename_axis("time")
+            .reset_index()
+            .melt(id_vars=["time"], var_name="bus", value_name="profile")
+            .assign(year=year_i, hydro_tech=technology)
+            .set_index(["bus", "time", "year", "hydro_tech"])
+            .to_xarray()
+        )
+
+        inflows.append(inflow)
+
+    ds = xr.merge(inflows)
+    ds = ds.sel(time=time)
+
+    ds.to_netcdf(snakemake.output.profile)