feat: reduce dependency to upstream retrieves

config/test/config.tyndp.yaml

@@ -437,7 +437,7 @@ cba:
   - 2040
   cba_scenario_input:
     use_presolved: false
-    sb_version: latest # currently only latest is supported
+    sb_version: latest # use 'latest' or a supported version from data/versions.csv for pre-solved SB network input in CBA; only applies if use_presolved is true
   methods:
   - toot
   - pint

data/tyndp_versions.csv

@@ -10,7 +10,6 @@ co2stop,26-august-2020,tyndp-archive,latest supported,2026-05-08,,https://storag
 copernicus_land_cover,v3.0.1,tyndp-archive,latest supported,2026-05-08,"The primary is already from Zenodo, documentation in https://zenodo.org/records/4723921",https://storage.googleapis.com/open-tyndp-data-store/archive/eur/copernicus_land_cover/v3.0.1
 corine,v18_5,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/corine/v18_5/corine.zip
 costs,v0.14.0,tyndp-archive,latest supported,2026-05-08,Part of the `technologydata` repository and versioned on GitHub.,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/costs/v0.14.0
-country_hdd,2025-08-13,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/country_hdd/2025-08-13/era5-HDD-per-country.csv
 country_runoff,2025-08-13,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/country_runoff/2025-08-13/era5-runoff-per-country.csv
 desnz_electricity_consumption,2026-02-16,tyndp-archive,latest supported,2026-02-16,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/desnz_electricity_consumption/2026-02-16/Subnational_electricity_consumption_statistics_2005-2024.xlsx
 dh_areas,341.5,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/dh_areas/341.5/dh_areas.gpkg
@@ -24,12 +23,9 @@ eurostat_balances,2026-02,tyndp-archive,latest supported,2026-05-08,,https://sto
 eurostat_household_balances,2025-07-09,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/eurostat_household_balances/2025-07-09/nrg_d_hhq.csv
 gdp_per_capita,2018-02-06,tyndp-archive,latest supported,2026-05-08,"Primary link is a direct download, earlier part of zenodo bundle",https://storage.googleapis.com/open-tyndp-data-store/archive/eur/gdp_per_capita/2018-02-06/GDP_per_capita_PPP_1990_2015_v2.nc
 gebco,2014,tyndp-archive,latest supported,2026-05-08,Earlier part of databundle,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/gebco/2014/GEBCO_2014_2D.nc
-gem_europe_gas_tracker,may-2024,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/gem_europe_gas_tracker/may-2024/Europe-Gas-Tracker-2024-05.xlsx
 gem_gcct,july-2025,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/gem_gcct/july-2025/Global-Cement-and-Concrete-Tracker_July-2025.xlsx
 gem_gspt,march-2025-v1.1,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/gem_gspt/march-2025-v1.1/Plant-level-data-Global-Iron-and-Steel-Tracker-March-2025-V1.1.xlsx
 geothermal_heat_utilisation_potentials,341.5,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/geothermal_heat_utilisation_potentials/341.5/Results_DH_Matching_Cluster.xlsx
-ghg_emissions,v23,tyndp-archive,latest supported,2026-05-08,earlier part of databundle,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/ghg_emissions/v23/UNFCCC_v23.csv
-h2_salt_caverns,16-10-2019,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/h2_salt_caverns/16-10-2019/h2_salt_caverns_GWh_per_sqkm.geojson
 hotmaps_industrial_sites,version-1.1,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/hotmaps_industrial_sites/version-1.1/Industrial_Database.csv
 jrc_ardeco,2021,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/jrc_ardeco/2021
 jrc_energy_atlas,2026-02-16,tyndp-archive,latest supported,2026-02-16,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/jrc_energy_atlas/2026-02-16/electricity_tot_demand_2019.tif
@@ -42,7 +38,6 @@ neso_electricity_demand,2026-02-02,tyndp-archive,latest supported,2026-05-08,,ht
 nitrogen_statistics,2023,tyndp-archive,latest supported,2026-02-26,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/nitrogen_statistics/2023/nitro-ert.xlsx
 nitrogen_statistics,2022,tyndp-archive,supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/nitrogen_statistics/2022/nitro-ert.xlsx
 nuts3_population,13-03-2025,tyndp-archive,latest supported,2026-05-08,earlier part of zenodo bundle,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/nuts3_population/13-03-2025/nama_10r_3popgdp.tsv.gz
-ons_lad,may-2024,tyndp-archive,latest supported,2026-02-17,API request used,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/ons_lad/may-2024/Local_Authority_Districts_May_2024_Boundaries__UK_BSC.geojson
 open_tyndp_prelim,0.7.1,primary,not-supported,2026-06-02,Open-TYNDP preliminary outcomes,https://zenodo.org/records/20303009
 open_tyndp_prelim,0.7.1,archive,latest supported,2026-06-02,Open-TYNDP preliminary outcomes (results-0.7.1.zip),https://zenodo.org/records/20303009/files/results-0.7.1.zip
 open_tyndp_prelim,0.7.1,tyndp-archive,latest supported,2026-06-02,Open-TYNDP preliminary outcomes (results-0.7.1.zip),https://storage.googleapis.com/open-tyndp-data-store/outcomes/0.7.1/results-0.7.1.zip
@@ -59,8 +54,6 @@ opsd_electricity_demand,2026-02-02,tyndp-archive,latest supported,2026-05-08,,ht
 osm,0.7,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/osm/0.7
 osm_boundaries,unknown,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/osm_boundaries/unknown/osm_boundaries.zip
 population_count,2018-11-01,tyndp-archive,latest supported,2026-05-08,earlier part of zenodo bundle,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/population_count/2018-11-01/ppp_2019_1km_Aggregated.tif
-powerplants,0.8.1,tyndp-archive,latest supported,2026-05-08,Part of the `powerplantmatching` package and versioned on GitHub.,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/powerplants/0.8.1/powerplants.csv
-scigrid_gas,1.1.2,tyndp-archive,latest supported,2026-05-08,the primary is already from Zenodo published by original authors,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/scigrid_gas/1.1.2/IGGIELGN.zip
 seawater_temperature,v1.0,tyndp-archive,latest supported,2026-05-08,Test cutout seawater temperature,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/seawater_temperature/v1.0/seawater_temperature.nc
 ship_raster,v5,tyndp-archive,latest supported,2026-05-08,,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/ship_raster/v5/shipdensity_global.zip
 synthetic_electricity_demand,v2,tyndp-archive,latest supported,2026-05-08,the primary is already from Zenodo published by original authors,https://storage.googleapis.com/open-tyndp-data-store/archive/eur/synthetic_electricity_demand/v2/demand_hourly.csv

doc/release_notes.md

@@ -80,6 +80,8 @@
 
 * Use `combine_indicators` instead of `collect_indicators` in the function `summary_benchmark_indicators` ([#797](https://github.com/open-energy-transition/open-tyndp/pull/797)).
 
+* Reduce dependency on upstream retrieves ([#798](https://github.com/open-energy-transition/open-tyndp/pull/798)).
+
 
 ## Upcoming PyPSA-Eur Release
 

rules/build_electricity.smk

@@ -816,7 +816,21 @@ rule add_electricity:
             f"costs_{config_provider('costs', 'year')(w)}_processed.csv"
         ),
         regions=resources("regions_onshore_base_s_{clusters}.geojson"),
-        powerplants=resources("powerplants_s_{clusters}.csv"),
+        powerplants=branch(
+            lambda w: config_provider("electricity", "conventional_carriers")(w)
+            or config_provider("electricity", "extendable_carriers", "Generator")(w)
+            or (
+                config_provider(
+                    "electricity", "estimate_renewable_capacities", "enable"
+                )(w)
+                and config_provider(
+                    "electricity",
+                    "estimate_renewable_capacities",
+                    "from_powerplantmatching",
+                )(w)
+            ),
+            resources("powerplants_s_{clusters}.csv"),
+        ),
         hydro_capacities=ancient("data/hydro_capacities.csv"),
         unit_commitment="data/unit_commitment.csv",
         fuel_price=lambda w: (

rules/build_sector.smk

@@ -830,7 +830,9 @@ rule build_swiss_energy_balances:
 rule build_energy_totals:
     input:
         nuts3_shapes=resources("nuts3_shapes.geojson"),
-        co2=rules.retrieve_ghg_emissions.output["csv"],
+        co2=branch(
+            config_provider("co2_budget"), rules.retrieve_ghg_emissions.output["csv"]
+        ),
         swiss=resources("switzerland_energy_balances.csv"),
         swiss_transport=f"{BFS_ROAD_VEHICLE_STOCK_DATASET['folder']}/vehicle_stock.csv",
         idees=rules.retrieve_jrc_idees.output["directory"],
@@ -1644,6 +1646,31 @@ def input_pemmdb_data(w):
     }
 
 
+def input_gas_network(w):
+    inputs = {}
+
+    gas_network = config_provider("sector", "gas_network")(w)
+    h2_retrofit = config_provider("sector", "H2_retrofit")(w)
+    imports_enabled = config_provider("sector", "imports", "enable")(w)
+    import_carriers = config_provider("sector", "imports", "carriers")(w)
+    h2_topology_tyndp = config_provider("sector", "h2_topology_tyndp")(w)
+
+    if gas_network or h2_retrofit:
+        inputs["clustered_gas_network"] = resources("gas_network_base_s_{clusters}.csv")
+
+    gas_imports = imports_enabled and "gas" in import_carriers
+    h2_imports_not_tyndp = (
+        imports_enabled and "H2" in import_carriers and not h2_topology_tyndp
+    )
+
+    if gas_network or gas_imports or h2_imports_not_tyndp:
+        inputs["gas_input_nodes_simplified"] = resources(
+            "gas_input_locations_s_{clusters}_simplified.csv"
+        )
+
+    return inputs
+
+
 def include_tyndp_projects(w):
     horizons = config_provider("tyndp_investment_candidates", "elec_projects")(w)
     if not horizons:
@@ -1667,8 +1694,7 @@ rule prepare_sector_network:
         unpack(input_heat_source_power),
         unpack(input_offshore_hubs),
         unpack(input_pemmdb_data),
-        **rules.cluster_gas_network.output,
-        **rules.build_gas_input_locations.output,
+        unpack(input_gas_network),
         retro_cost=lambda w: (
             resources("retro_cost_base_s_{clusters}.csv")
             if config_provider("sector", "heating")(w)
@@ -1699,7 +1725,10 @@ rule prepare_sector_network:
         pop_weighted_energy_totals=resources(
             "pop_weighted_energy_totals_s_{clusters}.csv"
         ),
-        pop_weighted_heat_totals=resources("pop_weighted_heat_totals_s_{clusters}.csv"),
+        pop_weighted_heat_totals=branch(
+            config_provider("sector", "heating"),
+            resources("pop_weighted_heat_totals_s_{clusters}.csv"),
+        ),
         shipping_demand=resources("shipping_demand_s_{clusters}.csv"),
         transport_demand=resources("transport_demand_s_{clusters}.csv"),
         transport_data=resources("transport_data_s_{clusters}.csv"),
@@ -1709,8 +1738,12 @@ rule prepare_sector_network:
             config_provider("sector", "residential_heat", "dsm", "enable"),
             resources("residential_heat_dsm_profile_total_base_s_{clusters}.csv"),
         ),
-        co2_totals_name=resources("co2_totals.csv"),
-        co2=rules.retrieve_ghg_emissions.output["csv"],
+        co2_totals_name=branch(
+            config_provider("co2_budget"), resources("co2_totals.csv")
+        ),
+        co2=branch(
+            config_provider("co2_budget"), rules.retrieve_ghg_emissions.output["csv"]
+        ),
         biomass_potentials=resources(
             "biomass_potentials_s_{clusters}_{planning_horizons}.csv"
         ),
@@ -1719,7 +1752,10 @@ rule prepare_sector_network:
             if config_provider("foresight")(w) == "overnight"
             else resources("costs_{planning_horizons}_processed.csv")
         ),
-        h2_cavern=resources("salt_cavern_potentials_s_{clusters}.csv"),
+        h2_cavern=branch(
+            config_provider("sector", "h2_topology_tyndp"),
+            otherwise=resources("salt_cavern_potentials_s_{clusters}.csv"),
+        ),
         busmap_s=resources("busmap_base_s.csv"),
         busmap=resources("busmap_base_s_{clusters}.csv"),
         clustered_pop_layout=resources("pop_layout_base_s_{clusters}.csv"),

rules/postprocess.smk

@@ -468,7 +468,9 @@ rule plot_summary:
         energy=RESULTS + "csvs/energy.csv",
         balances=RESULTS + "csvs/energy_balance.csv",
         eurostat=resources("eurostat_energy_balances.csv"),
-        co2=rules.retrieve_ghg_emissions.output["csv"],
+        co2=branch(
+            config_provider("co2_budget"), rules.retrieve_ghg_emissions.output["csv"]
+        ),
     output:
         costs=RESULTS + "graphs/costs.pdf",
         energy=RESULTS + "graphs/energy.pdf",

rules/solve_myopic.smk

@@ -10,7 +10,11 @@ rule add_existing_baseyear:
         network=resources(
             "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc"
         ),
-        powerplants=resources("powerplants_s_{clusters}.csv"),
+        powerplants=branch(
+            lambda w: config_provider("electricity", "conventional_carriers")(w)
+            or config_provider("electricity", "extendable_carriers", "Generator")(w),
+            resources("powerplants_s_{clusters}.csv"),
+        ),
         costs=lambda w: resources(
             f"costs_{config_provider('scenario', 'planning_horizons',0)(w)}_processed.csv"
         ),

rules/solve_perfect.smk

@@ -8,7 +8,11 @@ rule add_existing_baseyear:
         network=resources(
             "networks/base_s_{clusters}_{opts}_{sector_opts}_{planning_horizons}.nc"
         ),
-        powerplants=resources("powerplants_s_{clusters}.csv"),
+        powerplants=branch(
+            lambda w: config_provider("electricity", "conventional_carriers")(w)
+            or config_provider("electricity", "extendable_carriers", "Generator")(w),
+            resources("powerplants_s_{clusters}.csv"),
+        ),
         busmap_s=resources("busmap_base_s.csv"),
         busmap=resources("busmap_base_s_{clusters}.csv"),
         clustered_pop_layout=resources("pop_layout_base_s_{clusters}.csv"),

scripts/add_electricity.py

@@ -571,6 +571,9 @@ def attach_wind_and_solar(
         Planning horizon for which renewable profiles should be added. Optional input defaults to None. If
         year is part of the coordinates and planning_horizon is None, then the first year is used by default.
     """
+    if ppl.empty:
+        return
+
     add_missing_carriers(n, carriers)
 
     if landfall_lengths is None:
@@ -717,6 +720,9 @@ def attach_conventional_generators(
     fuel_price : pd.DataFrame, optional
         DataFrame containing fuel price data, by default None.
     """
+    if ppl.empty:
+        return
+
     carriers = list(
         set(conventional_carriers)
         | set(extendable_carriers["Generator"]) - set(renewable_carriers)
@@ -808,6 +814,9 @@ def attach_existing_batteries(
     ppl: pd.DataFrame,
 ) -> None:
     """Attach existing battery storage units from the power plant dataset."""
+    if ppl.empty:
+        return
+
     batt = ppl.query('carrier == "battery"')
     if batt.empty:
         return
@@ -868,6 +877,9 @@ def attach_hydro(
     **params :
         Additional parameters for hydro units.
     """
+    if ppl.empty:
+        return
+
     add_missing_carriers(n, carriers)
     add_co2_emissions(n, costs, carriers)
 
@@ -1342,13 +1354,16 @@ def attach_stores(
 
     costs = load_costs(snakemake.input.costs)
 
-    ppl = load_and_aggregate_powerplants(
-        snakemake.input.powerplants,
-        costs,
-        params.consider_efficiency_classes,
-        params.aggregation_strategies,
-        params.exclude_carriers,
-    )
+    if ppl_fn := snakemake.input.powerplants:
+        ppl = load_and_aggregate_powerplants(
+            ppl_fn,
+            costs,
+            params.consider_efficiency_classes,
+            params.aggregation_strategies,
+            params.exclude_carriers,
+        )
+    else:
+        ppl = pd.DataFrame()
 
     if snakemake.params.load_source != "tyndp":
         logger.info(

scripts/add_existing_baseyear.py

@@ -208,6 +208,9 @@ def add_power_capacities_installed_before_baseyear(
     """
     logger.debug(f"Adding power capacities installed before {baseyear}")
 
+    if not powerplants_file:
+        return
+
     df_agg = pd.read_csv(powerplants_file, index_col=0)
 
     rename_fuel = {

scripts/build_energy_totals.py

@@ -1366,13 +1366,16 @@ def build_heating_efficiencies(
     )
     district_heat_share.to_csv(snakemake.output.district_heat_share)
 
-    base_year_emissions = params["base_emissions_year"]
-    emissions_scope = snakemake.params.energy["emissions"]
-    eea_co2 = build_eea_co2(snakemake.input.co2, base_year_emissions, emissions_scope)
-    eurostat_co2 = build_eurostat_co2(eurostat, base_year_emissions)
-
-    co2 = build_co2_totals(countries, eea_co2, eurostat_co2)
-    co2.to_csv(snakemake.output.co2_name)
+    if co2_fn := snakemake.input.co2:
+        base_year_emissions = params["base_emissions_year"]
+        emissions_scope = snakemake.params.energy["emissions"]
+        eea_co2 = build_eea_co2(co2_fn, base_year_emissions, emissions_scope)
+        eurostat_co2 = build_eurostat_co2(eurostat, base_year_emissions)
+
+        co2 = build_co2_totals(countries, eea_co2, eurostat_co2)
+        co2.to_csv(snakemake.output.co2_name)
+    else:
+        pd.DataFrame().to_csv(snakemake.output.co2_name)
 
     transport = build_transport_data(countries, population, idees)
     transport.to_csv(snakemake.output.transport_name)

scripts/prepare_sector_network.py

@@ -9799,13 +9799,14 @@ def add_hydro_tyndp(
     pop_weighted_energy_totals = (
         pd.read_csv(snakemake.input.pop_weighted_energy_totals, index_col=0) * nyears
     )
-    pop_weighted_heat_totals = (
-        pd.read_csv(snakemake.input.pop_weighted_heat_totals, index_col=0) * nyears
-    )
-    pop_weighted_energy_totals.update(pop_weighted_heat_totals)
+    if options["heating"]:
+        pop_weighted_heat_totals = (
+            pd.read_csv(snakemake.input.pop_weighted_heat_totals, index_col=0) * nyears
+        )
+        pop_weighted_energy_totals.update(pop_weighted_heat_totals)
 
-    fn = snakemake.input.gas_input_nodes_simplified
-    gas_input_nodes = pd.read_csv(fn, index_col=0)
+    fn = snakemake.input.get("gas_input_nodes_simplified")
+    gas_input_nodes = pd.read_csv(fn, index_col=0) if fn else None
 
     if options.get("keep_existing_capacities", False):
         existing_capacities, existing_efficiencies = get_capacities_from_elec(
@@ -9977,7 +9978,7 @@ def add_hydro_tyndp(
         h2_pipes_file=snakemake.input.h2_grid_tyndp,
         interzonal_file=snakemake.input.interzonal_prepped,
         cavern_types=snakemake.params.sector["hydrogen_underground_storage_locations"],
-        clustered_gas_network_file=snakemake.input.clustered_gas_network,
+        clustered_gas_network_file=snakemake.input.get("clustered_gas_network"),
         gas_input_nodes=gas_input_nodes,
         spatial=spatial,
         options=options,
@@ -10232,8 +10233,8 @@ def add_hydro_tyndp(
     add_co2limit(
         n,
         options,
-        snakemake.input.co2_totals_name,
-        snakemake.params.countries,
+        snakemake.input.get("co2_totals_name"),
+        snakemake.params.get("countries"),
         nyears,
         limit,
     )