From 11f66b1e027bd76453cbe56ce41505a6564d69d2 Mon Sep 17 00:00:00 2001 From: lisa Date: Fri, 24 Jul 2026 16:31:51 +0200 Subject: [PATCH 1/3] feat: add weather years to planning horizon --- config/config.default.yaml | 1 + config/config.tyndp.yaml | 17 +++++++++++++++++ config/schema.default.json | 20 ++++++++++++++++++++ scripts/lib/validation/config/load.py | 4 ++++ 4 files changed, 42 insertions(+) diff --git a/config/config.default.yaml b/config/config.default.yaml index e4cb733ef0..13fdfda476 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -573,6 +573,7 @@ load: - 2030 - 2040 - 2050 + weather_year_tyndp: {} manual_adjustments: true scaling_factor: 1.0 fixed_year: false diff --git a/config/config.tyndp.yaml b/config/config.tyndp.yaml index 5fe0e820e6..c8c278d2c9 100644 --- a/config/config.tyndp.yaml +++ b/config/config.tyndp.yaml @@ -177,6 +177,23 @@ load: - 2030 - 2040 - 2050 + weather_year_tyndp: + 2030: + - 3 + - 21 + - 29 + 2035: + - 32 + - 37 + - 59 + 2040: + - 65 + - 71 + - 77 + 2050: + - 91 + - 92 + - 106 manual_adjustments: false supplement_synthetic: false patch_demand_with_mm: true diff --git a/config/schema.default.json b/config/schema.default.json index f42b05a74d..60145a44f3 100644 --- a/config/schema.default.json +++ b/config/schema.default.json @@ -3422,6 +3422,16 @@ }, "type": "array" }, + "weather_year_tyndp": { + "additionalProperties": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "description": "Mapping of planning horizons to the weather years (climate year column indices, e.g. 3 for `WS003`) that contain data in the TYNDP 2026 demand data, for that planning horizon. The first entry is used as the preferred weather year; if it's missing for a given demand type/file, the script falls back to another entry in the list (see `scripts._helpers.check_weather_year`).", + "type": "object" + }, "manual_adjustments": { "default": true, "description": "Whether to adjust the load data manually according to the function in `manual_adjustment`.", @@ -11736,6 +11746,16 @@ }, "type": "array" }, + "weather_year_tyndp": { + "additionalProperties": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "description": "Mapping of planning horizons to the weather years (climate year column indices, e.g. 3 for `WS003`) that contain data in the TYNDP 2026 demand data, for that planning horizon. The first entry is used as the preferred weather year; if it's missing for a given demand type/file, the script falls back to another entry in the list (see `scripts._helpers.check_weather_year`).", + "type": "object" + }, "manual_adjustments": { "default": true, "description": "Whether to adjust the load data manually according to the function in `manual_adjustment`.", diff --git a/scripts/lib/validation/config/load.py b/scripts/lib/validation/config/load.py index 0afb78c48c..85d0d815b1 100644 --- a/scripts/lib/validation/config/load.py +++ b/scripts/lib/validation/config/load.py @@ -61,6 +61,10 @@ class LoadConfig(BaseModel): default_factory=lambda: [2030, 2040, 2050], description="List of years for which TYNDP demand data is available.", ) + weather_year_tyndp: dict[int, list[int]] = Field( + default_factory=dict, + description="Mapping of planning horizons to the weather years (climate year column indices, e.g. 3 for `WS003`) that contain data in the TYNDP 2026 demand data, for that planning horizon. The first entry is used as the preferred weather year; if it's missing for a given demand type/file, the script falls back to another entry in the list (see `scripts._helpers.check_weather_year`).", + ) manual_adjustments: bool = Field( True, description="Whether to adjust the load data manually according to the function in `manual_adjustment`.", From ee99b8604b0ad4ef0084685825c8e9ec8a1d2c40 Mon Sep 17 00:00:00 2001 From: lisa Date: Fri, 24 Jul 2026 16:33:32 +0200 Subject: [PATCH 2/3] feat: add weather years to planning horizon in test config --- config/test/config.tyndp.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/config/test/config.tyndp.yaml b/config/test/config.tyndp.yaml index f62e2f2da9..684c4ae7f8 100644 --- a/config/test/config.tyndp.yaml +++ b/config/test/config.tyndp.yaml @@ -184,6 +184,23 @@ load: - 2030 - 2040 - 2050 + weather_year_tyndp: + 2030: + - 3 + - 21 + - 29 + 2035: + - 32 + - 37 + - 59 + 2040: + - 65 + - 71 + - 77 + 2050: + - 91 + - 92 + - 106 manual_adjustments: false supplement_synthetic: false patch_demand_with_mm: true From 4b38f7cd758c9deab372bd886b5301fad8d3306b Mon Sep 17 00:00:00 2001 From: lisa Date: Fri, 24 Jul 2026 16:35:33 +0200 Subject: [PATCH 3/3] feat: add new script processing all input data --- rules/sb.smk | 205 ++++++++++++++++++++++++ scripts/_helpers.py | 42 +++++ scripts/sb/build_tyndp_demand.py | 266 +++++++++++++++++++++++++++++++ 3 files changed, 513 insertions(+) create mode 100644 scripts/sb/build_tyndp_demand.py diff --git a/rules/sb.smk b/rules/sb.smk index 6f6ae4be97..4b2bf22d29 100644 --- a/rules/sb.smk +++ b/rules/sb.smk @@ -202,6 +202,211 @@ use rule build_electricity_demand as build_electricity_demand_tyndp with: benchmarks("performances/build_electricity_demand_{planning_horizons}") +def get_weather_year_tyndp(w): + """Get the preferred TYNDP 2026 weather year (climate year column index) for a given planning horizon.""" + weather_years = config_provider("load", "weather_year_tyndp")(w) + pyear = safe_pyear( + w.planning_horizons, + available_years=sorted(weather_years), + source="TYNDP demand weather year", + verbose=False, + ) + return weather_years[pyear][0] + + +# Generic rule: parameterized by the `demand_type` wildcard, so any demand +# type/file present under `data/tyndp_2026_bundle/Demand` can be requested +# directly by target filename. The concrete demand types below (electricity +# market, EV charging, hydrogen zones, ...) are defined as named aliases of +# this rule via `use rule ... as ...`, fixing `demand_type` and giving each a +# stable, readable output name instead of relying on the wildcard. +rule build_tyndp_demand: + input: + # TODO Replace with rules.retrieve_tyndp_2026.output.demand once available + demand="data/tyndp_2026_bundle/Demand", + output: + demand=resources("demand_tyndp_{demand_type}_{planning_horizons}.csv"), + log: + logs("build_tyndp_demand_{demand_type}_{planning_horizons}.log"), + benchmark: + benchmarks("performances/build_tyndp_demand_{demand_type}_{planning_horizons}") + threads: 1 + resources: + mem_mb=4000, + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type=lambda w: w.demand_type, + script: + scripts("sb/build_tyndp_demand.py") + + +use rule build_tyndp_demand as build_tyndp_electricity_market_demand with: + output: + demand=resources("electricity_demand_tyndp_{planning_horizons}.csv"), + log: + logs("build_tyndp_electricity_market_demand_{planning_horizons}.log"), + benchmark: + benchmarks( + "performances/build_tyndp_electricity_market_demand_{planning_horizons}" + ) + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="ELECTRICITY_MARKET", + + +use rule build_tyndp_demand as build_tyndp_electricity_prosumer_demand with: + output: + demand=resources("electricity_prosumer_demand_tyndp_{planning_horizons}.csv"), + log: + logs("build_tyndp_electricity_prosumer_demand_{planning_horizons}.log"), + benchmark: + benchmarks( + "performances/build_tyndp_electricity_prosumer_demand_{planning_horizons}" + ) + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="ELECTRICITY_PROSUMER", + + +use rule build_tyndp_demand as build_tyndp_electricity_prosumer_btm_demand with: + output: + demand=resources( + "electricity_prosumer_btm_demand_tyndp_{planning_horizons}.csv" + ), + log: + logs("build_tyndp_electricity_prosumer_btm_demand_{planning_horizons}.log"), + benchmark: + benchmarks( + "performances/build_tyndp_electricity_prosumer_btm_demand_{planning_horizons}" + ) + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="ELECTRICITY_PROSUMER_BEHIND_THE_METER_FIXED_LOAD", + + +use rule build_tyndp_demand as build_tyndp_ev_market_demand with: + output: + demand=resources("ev_market_demand_tyndp_{planning_horizons}.csv"), + log: + logs("build_tyndp_ev_market_demand_{planning_horizons}.log"), + benchmark: + benchmarks("performances/build_tyndp_ev_market_demand_{planning_horizons}") + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="EV_FIXED_LOAD_PROFILES_ELECTRICITY_MARKET", + + +use rule build_tyndp_demand as build_tyndp_ev_prosumer_demand with: + output: + demand=resources("ev_prosumer_demand_tyndp_{planning_horizons}.csv"), + log: + logs("build_tyndp_ev_prosumer_demand_{planning_horizons}.log"), + benchmark: + benchmarks("performances/build_tyndp_ev_prosumer_demand_{planning_horizons}") + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="EV_FIXED_LOAD_PROFILES_ELECTRICITY_PROSUMER", + + +use rule build_tyndp_demand as build_tyndp_h2_zone1_demand with: + output: + demand=resources("h2_zone1_demand_tyndp_{planning_horizons}.csv"), + log: + logs("build_tyndp_h2_zone1_demand_{planning_horizons}.log"), + benchmark: + benchmarks("performances/build_tyndp_h2_zone1_demand_{planning_horizons}") + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="Hydrogen_Zone 1", + + +use rule build_tyndp_demand as build_tyndp_h2_zone2_demand with: + output: + demand=resources("h2_zone2_demand_tyndp_{planning_horizons}.csv"), + log: + logs("build_tyndp_h2_zone2_demand_{planning_horizons}.log"), + benchmark: + benchmarks("performances/build_tyndp_h2_zone2_demand_{planning_horizons}") + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="Hydrogen_Zone 2", + + +use rule build_tyndp_demand as build_tyndp_synthetic_fuels_demand with: + output: + demand=resources("synthetic_fuels_demand_tyndp_{planning_horizons}.csv"), + log: + logs("build_tyndp_synthetic_fuels_demand_{planning_horizons}.log"), + benchmark: + benchmarks( + "performances/build_tyndp_synthetic_fuels_demand_{planning_horizons}" + ) + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="SYNTHETIC_FUELS", + + +use rule build_tyndp_demand as build_tyndp_thermal_hydrogen_demand with: + output: + demand=resources("thermal_hydrogen_demand_tyndp_{planning_horizons}.csv"), + log: + logs("build_tyndp_thermal_hydrogen_demand_{planning_horizons}.log"), + benchmark: + benchmarks( + "performances/build_tyndp_thermal_hydrogen_demand_{planning_horizons}" + ) + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="Thermal_energy_Hydrogen", + + +use rule build_tyndp_demand as build_tyndp_thermal_methane_demand with: + output: + demand=resources("thermal_methane_demand_tyndp_{planning_horizons}.csv"), + log: + logs("build_tyndp_thermal_methane_demand_{planning_horizons}.log"), + benchmark: + benchmarks( + "performances/build_tyndp_thermal_methane_demand_{planning_horizons}" + ) + params: + snapshots=config_provider("snapshots"), + drop_leap_day=config_provider("enable", "drop_leap_day"), + weather_year=get_weather_year_tyndp, + weather_years=config_provider("load", "weather_year_tyndp"), + demand_type="Thermal_energy_Methane", + + def get_pecd_prebuilt(w): if "pre-built" in PECD_DATASET["version"]: return rules.retrieve_tyndp_pecd.output.dir diff --git a/scripts/_helpers.py b/scripts/_helpers.py index e5418a22d9..adcf686d76 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -1546,6 +1546,48 @@ def check_cyear(cyear: int, scenario: str) -> int: return cyear +def check_weather_year(weather_year: int, valid_weather_years: list[int]) -> int: + """ + Check if the weather year is one of the known-valid ones, falling back if not. + + TYNDP 2026 demand profiles provide 30 climate year columns per planning + horizon (labelled ``WSxxx``), but depending on the demand type, either all + of them contain data or only a handful of them do (the rest being empty + placeholders). Which weather years are valid therefore has to be + determined per file (see e.g. ``get_valid_weather_years`` in + `scripts/sb/build_tyndp_demand.py`) rather than assumed globally. + + Parameters + ---------- + weather_year : int + Weather year (climate year column index, e.g. 3 for ``WS003``) to validate. + valid_weather_years : list[int] + Weather years known to contain data, for the file being processed. + + Returns + ------- + int + Valid weather year, falling back to the first entry of + `valid_weather_years` if the input is not among them. + """ + + if not valid_weather_years: + raise ValueError( + "No `valid_weather_years` provided. Expected a non-empty list of weather years." + ) + + if weather_year not in valid_weather_years: + fallback = valid_weather_years[0] + logger.warning( + f"Weather year WS{weather_year:03d} doesn't contain data for this file " + f"(available: {[f'WS{y:03d}' for y in valid_weather_years]}). " + f"Falling back to WS{fallback:03d}." + ) + weather_year = fallback + + return weather_year + + def get_tyndp_conventional_thermals( mapping: pd.DataFrame, tyndp_conventional_carriers: list[str], diff --git a/scripts/sb/build_tyndp_demand.py b/scripts/sb/build_tyndp_demand.py new file mode 100644 index 0000000000..268c96baf1 --- /dev/null +++ b/scripts/sb/build_tyndp_demand.py @@ -0,0 +1,266 @@ +# SPDX-FileCopyrightText: Contributors to Open-TYNDP +# +# SPDX-License-Identifier: MIT +""" +Builds TYNDP Scenario Building demand profiles for Open-TYNDP. + +This script processes demand data from TYNDP 2026 (e.g. electricity market, +electricity prosumer, EV charging, hydrogen zones, thermal energy, synthetic +fuels, ...), using the `weather_year` parameter to select one of the available +climate year columns for demand profiles. + +Weather Year Selection +----------------------- + +Each planning horizon Excel file contains 30 climate year columns (labelled +``WS001``-``WS030``, ``WS031``-``WS060``, etc.), only 3 of which contain data +for the corresponding planning horizon (the rest are empty/zero-filled +placeholders). Which 3 are populated is the same across demand types for a +given planning horizon, and is configured via `load.weather_year_tyndp` +(mapping planning horizon -> list of valid weather years). `weather_year` +selects which of these to use (e.g. `3` for `WS003`); if it isn't valid for +the requested planning horizon, the script falls back to the first entry of +`weather_years` for that horizon (see +:py:func:`scripts._helpers.check_weather_year`). + +Data Availability +----------------- + +Demand data is available for 2030, 2035, 2040 and 2050. Missing planning +horizons are linearly interpolated between available data points. + +Inputs +------ + +- `data/tyndp_2026_bundle/Demand`: TYNDP 2026 demand profiles, with one + subfolder per planning horizon containing one Excel file per demand type + (e.g. ``ELECTRICITY_MARKET {pyear}.xlsx``, ``Hydrogen_Zone 1_{pyear}.xlsx``), + each with one sheet per node. + +Outputs +------- + +- `resources/demand_tyndp_{demand_type}_{planning_horizons}.csv`: Processed + demand time series for the specified demand type and planning horizon +""" + +import logging +from pathlib import Path + +import pandas as pd + +from scripts._helpers import ( + align_demand_to_snapshots, + check_weather_year, + configure_logging, + get_snapshots, + interpolate_demand, + set_scenario_config, +) + +logger = logging.getLogger(__name__) + +# Arbitrary non-leap placeholder year used to build a DatetimeIndex from the +# demand data's day/hour index. Kept fixed across planning years so that +# interpolation between two planning years aligns on matching dates; the +# actual target year is applied later by `align_demand_to_snapshots`. +REFERENCE_YEAR = 2013 + + +def multiindex_to_datetimeindex(df: pd.DataFrame, year: int) -> pd.DataFrame: + """Convert demand MultiIndex ('Date', 'Hour') to a DatetimeIndex and return a DataFrame.""" + + df_reset = df.reset_index() + + df_reset["datetime"] = pd.to_datetime( + df_reset["Date"].str.strip(".") + + f".{year} " + + (df_reset["Hour"] - 1).astype(str) + + ":00", + format="%d.%m.%Y %H:%M", + ) + + # Set as index and drop the old columns + df_new = df_reset.set_index("datetime").drop(columns=["Date", "Hour"]) + + return df_new + + +def get_available_years(fn: str) -> list[int]: + """Scan the directory to find which planning years are available.""" + return sorted( + int(folder.name) + for folder in Path(fn).iterdir() + if folder.is_dir() and folder.name.isdigit() + ) + + +def get_file_path(fn: str, pyear: int, demand_type: str) -> Path: + """ + Construct file path to the demand Excel file for a given planning year and demand type. + + Filenames follow the pattern ``{demand_type}[ |_]{pyear}.xlsx`` (the + separator before the year isn't consistent across demand types), so the + file is looked up by matching on the `demand_type` prefix instead of a + fixed template. + """ + pyear_dir = Path(fn, str(pyear)) + matches = sorted(pyear_dir.glob(f"{demand_type}*{pyear}.xlsx")) + + if not matches: + raise FileNotFoundError( + f"No demand file found for demand type '{demand_type}' in {pyear_dir}" + ) + if len(matches) > 1: + logger.warning( + f"Multiple demand files match demand type '{demand_type}' in {pyear_dir}: " + f"{[m.name for m in matches]}. Using {matches[0].name}." + ) + + return matches[0] + + +def read_demand_excel(demand_fn: str, weather_year: int) -> pd.DataFrame: + """Read and process demand data from Excel file for a specific weather year.""" + try: + ws_code = f"WS{weather_year:03d}" + data = pd.read_excel( + demand_fn, + header=10, + index_col=[0, 1], + sheet_name=None, + usecols=lambda name: name == "Date" or name == "Hour" or name == ws_code, + engine="calamine", + ) + + demand = pd.concat(data, axis=1).droplevel(1, axis=1) + # Reindex to match snapshots + demand = multiindex_to_datetimeindex(demand, year=REFERENCE_YEAR) + # Rename UK in GB + demand.columns = demand.columns.str.replace("UK", "GB") + demand.columns.name = "Bus" + + except Exception as e: + logger.warning( + f"Failed to read demand from {demand_fn}, weather year {ws_code}: " + f"{type(e).__name__}: {e}" + ) + demand = pd.DataFrame() + + return demand + + +def load_single_year( + fn: str, + pyear: int, + weather_year: int, + demand_type: str, + weather_years: dict[int, list[int]], +) -> pd.DataFrame: + """Load demand data for a single planning year.""" + demand_fn = get_file_path(fn, pyear, demand_type) + + valid_weather_years = weather_years.get(pyear) + if valid_weather_years is None: + logger.warning( + f"No valid weather years configured for planning year {pyear}. " + f"Using requested weather year WS{weather_year:03d} as is." + ) + valid_weather_years = [weather_year] + + weather_year = check_weather_year(weather_year, valid_weather_years) + return read_demand_excel(demand_fn, weather_year) + + +def load_demand( + fn: str, + pyear: int, + weather_year: int, + demand_type: str, + weather_years: dict[int, list[int]], +) -> pd.DataFrame: + """ + Load demand data for a specific planning year, weather year and demand type. + + This function retrieves demand data from a file, either by loading the + exact year if available or by performing linear interpolation between + available years. The data is filtered for a specific weather year. + + Parameters + ---------- + fn : str + Filepath to the demand data directory. + pyear : int + Planning year for which to retrieve demand data. + weather_year : int + Weather year used to filter the demand data. + demand_type : str + Demand type to load (e.g. "ELECTRICITY_MARKET", "Hydrogen_Zone 1"). + weather_years : dict[int, list[int]] + Mapping of planning horizon to the weather years known to contain + data for that horizon (see `load.weather_year_tyndp` in the config). + + Returns + ------- + pd.DataFrame + DataFrame containing demand data for the specified planning year, + weather year and demand type. + """ + + available_years = get_available_years(fn) + logger.info(f"Available years: {available_years}, Target year: {pyear}") + + # If target year exists in data, load it directly + if pyear in available_years: + logger.info(f"Year {pyear} found in available data. Loading directly.") + return load_single_year(fn, pyear, weather_year, demand_type, weather_years) + + # Target year not available, do linear interpolation + return interpolate_demand( + available_years=available_years, + pyear=pyear, + load_single_year_func=load_single_year, + fn=fn, + weather_year=weather_year, + demand_type=demand_type, + weather_years=weather_years, + ) + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from scripts._helpers import mock_snakemake + + snakemake = mock_snakemake( + "build_tyndp_demand", + planning_horizons="2040", + demand_type="ELECTRICITY_MARKET", + clusters="all", + configfiles="config/test/config.tyndp.yaml", + ) + + configure_logging(snakemake) + set_scenario_config(snakemake) + + # Parameters + pyear = int(snakemake.wildcards.planning_horizons) + demand_type = snakemake.params.demand_type + weather_year = int(snakemake.params.weather_year) + weather_years = { + int(y): list(ws) for y, ws in snakemake.params.weather_years.items() + } + snapshots = get_snapshots( + snakemake.params.snapshots, snakemake.params.drop_leap_day + ) + fn = snakemake.input.demand + + logger.info( + f"Processing '{demand_type}' demand for target year: {pyear}, weather year: WS{weather_year:03d}" + ) + demand = load_demand(fn, pyear, weather_year, demand_type, weather_years) + + # Reindex demand to fit to snapshots + demand = align_demand_to_snapshots(demand, snapshots) + + # Export to CSV + demand.to_csv(snakemake.output.demand, index=True)