open-energy-transition
diff --git a/‎.github/workflows/test.yaml
+7 b/‎.github/workflows/test.yaml
+7
diff --git a/‎.github/workflows/update-pinned-env.yaml
+6-2 b/‎.github/workflows/update-pinned-env.yaml
+6-2
diff --git a/‎.github/workflows/validate.yaml
+15-1 b/‎.github/workflows/validate.yaml
+15-1
diff --git a/‎.pre-commit-config.yaml
+2-2 b/‎.pre-commit-config.yaml
+2-2
diff --git a/‎Snakefile
+12-4 b/‎Snakefile
+12-4
diff --git a/‎config/config.default.yaml
+33-16 b/‎config/config.default.yaml
+33-16
diff --git a/‎config/test/config.electricity.yaml
+2-2 b/‎config/test/config.electricity.yaml
+2-2
diff --git a/‎config/test/config.myopic.yaml
+1-1 b/‎config/test/config.myopic.yaml
+1-1
diff --git a/‎config/test/config.overnight.yaml
+3-3 b/‎config/test/config.overnight.yaml
+3-3
diff --git a/‎config/test/config.perfect.yaml
+3-3 b/‎config/test/config.perfect.yaml
+3-3
diff --git a/‎config/test/config.scenarios.yaml
+1-1 b/‎config/test/config.scenarios.yaml
+1-1
@@ -70,7 +70,9 @@ jobs:
 
     - uses: conda-incubator/setup-miniconda@v3
       with:
+        miniforge-version: latest
         activate-environment: pypsa-eur
+        channel-priority: strict
 
     - name: Cache Conda env
       uses: actions/cache@v4
@@ -85,6 +87,11 @@ jobs:
         conda env update -n pypsa-eur -f ${{ env.env_file }}
         echo "Run conda list" && conda list
 
+    - name: Run pylint check on scripts
+      # check for undefined variables to reuse functions across scripts
+      run: |
+        pylint --disable=all --enable=E0601 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_* 
+
     - name: Run snakemake test workflows
       run: |
         make test
 
@@ -34,8 +34,9 @@ jobs:
     - name: Setup conda
       uses: conda-incubator/setup-miniconda@v3
       with:
+        miniforge-version: latest
         activate-environment: ${{ github.event.repository.name }}
-        channels: bioconda,conda-forge,defaults
+        channel-priority: strict
         environment-file: ${{ env.BASE_ENV }}
 
     - name: Update pinned environment per OS
@@ -83,5 +84,8 @@ jobs:
         token: ${{ secrets.GITHUB_TOKEN }}
         branch: update-pinned-environment
         title: "[github-actions.ci] Update pinned envs"
-        body: "Automatically generated PR to update pinned environment files for Windows, macOS, and Linux."
+        body: |
+          Automatically generated PR to update pinned environment files for Windows, macOS, and Linux.
+
+          **Note: Do not merge without manual test execution. Either update the branch to trigger tests, or use `workflow_dispatch` to run tests manually. Unlike standard PRs, tests will not run automatically.**
         commit-message: "Update pinned environment files for all platforms"
@@ -14,19 +14,33 @@ jobs:
     name: Run validation
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: self-hosted
+    outputs:
+      validation_failed: ${{ steps.check_validation.outputs.failed }}
     steps:
     - uses: lkstrp/[email protected]
+      id: validation
+      continue-on-error: true
       with:
         step: run-self-hosted-validation
         env_file: envs/environment.yaml
         snakemake_config: config/test/config.validator.yaml
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Check validation status
+      id: check_validation
+      run: |
+        if [[ "${{ steps.validation.outcome }}" == "failure" ]]; then
+          echo "failed=true" >> $GITHUB_OUTPUT
+        else
+          echo "failed=false" >> $GITHUB_OUTPUT
+        fi
 
   create-report:
     name: Create report
-    if: github.event.pull_request.head.repo.full_name == github.repository
     needs: run-validation
+    if: |
+      github.event.pull_request.head.repo.full_name == github.repository && 
+      needs.run-validation.outputs.validation_failed != 'true'
     runs-on: ubuntu-latest
     steps:
     - uses: lkstrp/[email protected]
 
@@ -14,7 +14,7 @@ repos:
 # Run ruff to lint and format
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.9.3
+  rev: v0.9.10
   hooks:
     # Run the linter.
   - id: ruff
@@ -24,7 +24,7 @@ repos:
 
   # Find common spelling mistakes in comments and docstrings
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.4.0
+  rev: v2.4.1
   hooks:
   - id: codespell
     args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor,pris'] # Ignore capital case words, e.g. country codes
 
@@ -4,13 +4,19 @@
 
 from pathlib import Path
 import yaml
-from os.path import normpath, exists
+from os.path import normpath, exists, join
 from shutil import copyfile, move, rmtree
 from snakemake.utils import min_version
 
 min_version("8.11")
 
-from scripts._helpers import path_provider, copy_default_files, get_scenarios, get_rdir
+from scripts._helpers import (
+    path_provider,
+    copy_default_files,
+    get_scenarios,
+    get_rdir,
+    get_shadow,
+)
 
 
 copy_default_files(workflow)
@@ -23,14 +29,16 @@ configfile: "config/config.yaml"
 run = config["run"]
 scenarios = get_scenarios(run)
 RDIR = get_rdir(run)
+shadow_config = get_shadow(run)
 
 shared_resources = run["shared_resources"]["policy"]
 exclude_from_shared = run["shared_resources"]["exclude"]
 logs = path_provider("logs/", RDIR, shared_resources, exclude_from_shared)
 benchmarks = path_provider("benchmarks/", RDIR, shared_resources, exclude_from_shared)
 resources = path_provider("resources/", RDIR, shared_resources, exclude_from_shared)
 
-CDIR = "" if run["shared_cutouts"] else RDIR
+cutout_dir = config["atlite"]["cutout_directory"]
+CDIR = join(cutout_dir, ("" if run["shared_cutouts"] else RDIR))
 RESULTS = "results/" + RDIR
 
 
@@ -39,7 +47,7 @@ localrules:
 
 
 wildcard_constraints:
-    clusters="[0-9]+(m|c)?|all",
+    clusters="[0-9]+(m|c)?|all|adm",
     ll=r"(v|c)([0-9\.]+|opt)",
     opts=r"[-+a-zA-Z0-9\.]*",
     sector_opts=r"[-+a-zA-Z0-9\.\s]*",
 
@@ -30,6 +30,7 @@ run:
     policy: false
     exclude: []
   shared_cutouts: true
+  use_shadow_directory: true # Set to false if problems regarding missing directories occur
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight
 foresight: overnight
@@ -67,7 +68,6 @@ enable:
   retrieve_cost_data: true
   build_cutout: false
   retrieve_cutout: true
-  custom_busmap: false
   drop_leap_day: true
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget
@@ -132,6 +132,7 @@ electricity:
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite
 atlite:
+  cutout_directory: cutouts
   default_cutout: europe-2013-sarah3-era5
   nprocesses: 4
   show_progress: false
@@ -480,16 +481,9 @@ sector:
       heat_exchanger_pinch_point_temperature_difference: 5 #K
       isentropic_compressor_efficiency: 0.8
       heat_loss: 0.0
-    heat_utilisation_potentials:
+    limited_heat_sources:
       geothermal:
-        # activate for 85C hydrothermal
-        # key: hydrothermal_85
-        # constant_temperature_celsius: 85
-        key: hydrothermal_65
         constant_temperature_celsius: 65
-        column_name: Energy_TWh
-        unit: TWh
-        full_load_hours: 4000
     direct_utilisation_heat_sources:
     - geothermal
   heat_pump_sources:
@@ -618,12 +612,15 @@ sector:
     micro_chp: false # Only gas is used for micro_chp
   solar_thermal: true
   solar_cf_correction: 0.788457  # =  >>> 1/1.2683
-  marginal_cost_storage: 0. #1e-4
   methanation: true
   coal_cc: false
   dac: true
   co2_vent: false
-  central_heat_vent: false
+  heat_vent:
+    urban central: true
+    urban decentral: true
+    rural: true
+  marginal_cost_heat_vent: 0.02
   allam_cycle_gas: false
   hydrogen_fuel_cell: true
   hydrogen_turbine: false
@@ -851,9 +848,7 @@ costs:
     lifetime: 25
     "CO2 intensity": 0
     "discount rate": 0.07
-  # Marginal and capital costs can be overwritten
-  # capital_cost:
-  #   onwind: 500
+  overwrites: {}
   marginal_cost:
     solar: 0.01
     onwind: 0.015
@@ -864,13 +859,18 @@ costs:
     fuel cell: 0.
     battery: 0.
     battery inverter: 0.
+    home battery storage: 0
+    water tank charger: 0.03
   emission_prices:
     enable: false
     co2: 0.
     co2_monthly_prices: false
 
 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering
 clustering:
+  mode: busmap
+  administrative:
+    level: 1
   focus_weights: false
   simplify_network:
     to_substations: false
@@ -964,6 +964,12 @@ solving:
       ipm_optimality_tolerance: 1e-4
       parallel: "on"
       random_seed: 123
+    highs-simplex:
+      solver: "simplex"
+      parallel: "on"
+      primal_feasibility_tolerance: 1e-5
+      dual_feasibility_tolerance: 1e-5
+      random_seed: 123
     gurobi-default:
       threads: 8
       method: 2 # barrier
@@ -1020,6 +1026,7 @@ solving:
     rtol: 0.01
 
   mem_mb: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2
+  memory_logging_frequency: 30 # in seconds
   runtime: 6h #runtime in humanfriendly style https://humanfriendly.readthedocs.io/en/latest/
 
 
@@ -1210,8 +1217,18 @@ plotting:
     services urban decentral water tanks: '#f1b4a4'
     urban central water tanks: '#e9977d'
     hot water storage: '#e69487'
-    water pits: '#cc826a'
-    urban central water pits: '#d96f4c'
+    hot water charging: '#e8998b'
+    urban central water tanks charger: '#b57a67'
+    residential rural water tanks charger: '#b4887c'
+    residential urban decentral water tanks charger: '#b39995'
+    services rural water tanks charger: '#b3abb0'
+    services urban decentral water tanks charger: '#b3becc'
+    hot water discharging: '#e99c8e'
+    urban central water tanks discharger: '#b9816e'
+    residential rural water tanks discharger: '#ba9685'
+    residential urban decentral water tanks discharger: '#baac9e'
+    services rural water tanks discharger: '#bbc2b8'
+    services urban decentral water tanks discharger: '#bdd8d3'
     # heat demand
     Heat load: '#cc1f1f'
     heat: '#cc1f1f'
 
@@ -80,10 +80,10 @@ lines:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex
 
   check_objective:
-    enable: false
+    enable: true
     expected_value: 3.8120188094e+07
 
 plotting:
 
@@ -85,7 +85,7 @@ industry:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex
   mem: 4000
 
 plotting:
 
@@ -79,12 +79,12 @@ industry:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex
   mem: 4000
 
   check_objective:
-    enable: false
-    expected_value: 7.0847670388e+08
+    enable: true
+    expected_value: 6.96e+08
 
 plotting:
   map:
 
@@ -84,12 +84,12 @@ industry:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex
   mem: 4000
 
   check_objective:
-    enable: false
-    expected_value: 1.4427662256e+10
+    enable: true
+    expected_value: 1.3778753459e+10
 
 plotting:
   map:
 
@@ -58,4 +58,4 @@ renewable:
 solving:
   solver:
     name: highs
-    options: highs-default
+    options: highs-simplex