Merge changes from upstream incl. data layer

.github/pull_request_template.md

@@ -17,16 +17,14 @@ Closes # (if applicable).
 
 ## Checklist
 
-<!-- Remove what doesn't apply. -->
-
 - [ ] I tested my contribution locally and it works as intended.
 - [ ] Code and workflow changes are sufficiently documented.
 - [ ] Changed dependencies are added to `pixi.toml` (using `pixi add <dependency-name>`).
 - [ ] Changes in configuration options are added in `config/config.default.yaml`.
 - [ ] Changes in configuration options are documented in `doc/configtables/*.csv`.
 - [ ] Changes in configuration options are added in `config/test/*.yaml`.
 - [ ] Open-TYNDP SPDX license header added to all touched files.
-- [ ] Sources of newly added data are documented in `doc/data_sources.rst`.
+- [ ] For new data sources or versions, [these instructions](https://pypsa-eur.readthedocs.io/en/latest/data_sources.html) have been followed.
 - [ ] New rules are documented in the appropriate `doc/*.rst` files.
 - [ ] A release note `doc/release_notes.rst` is added.
 - [ ] Major features are documented with up-to-date information in `README` and `doc/index.rst`.

.github/workflows/test.yaml

@@ -30,6 +30,23 @@ jobs:
         shell: bash -l {0}
 
     steps:
+    - uses: actions/checkout@v6
+
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        filters: |
+          src:
+            - 'scripts/**'
+            - 'rules/**'
+            - 'data/**'
+            - 'Snakefile'
+            - 'config/**'
+            - 'test/**'
+            - 'pixi.toml'
+            - 'pixi.lock'
+            - '.github/workflows/test.yaml'
+
     - name: Free up disk space
       run: |
         echo "Initial disk space"
@@ -40,10 +57,12 @@ jobs:
         sudo docker builder prune -a --force
         echo "Final disk space"
         df -h
-
-    - uses: actions/checkout@v6
+    - name: Skip - no source changes
+      if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule'
+      run: echo "Skipping tests because no source code changes detected"
 
     - name: Setup Pixi
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       uses: prefix-dev/setup-pixi@v0.9.3
       with:
         pixi-version: v0.59.0
@@ -53,49 +72,56 @@ jobs:
         environments: open-tyndp
 
     - name: Setup cache keys
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: |
         echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
 
-    - uses: actions/cache@v4
+    - uses: actions/cache@v5
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       with:
         path: |
           data
           cutouts
         key: data-cutouts-${{ env.WEEK }}
 
     - name: Run pylint check on scripts
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       # check for undefined variables to reuse functions across scripts
       run: |
         pixi run pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*
 
     - name: Run snakemake test workflows (sb)
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
         pixi run -e open-tyndp tyndp-sb-test
 
     - name: Run snakemake test workflows (cba)
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
         pixi run -e open-tyndp tyndp-cba-test
 
     - name: Run snakemake test workflows (upstream)
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
         pixi run -e open-tyndp integration-tests
 
     - name: Run unit tests
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: |
         pixi run -e open-tyndp unit-tests
 
     - name: Upload artifacts
       if: always()
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: results-${{ matrix.os }}
         path: |

.github/workflows/update-lockfile.yaml

@@ -44,7 +44,7 @@ jobs:
         pixi workspace export conda-environment -e default envs/environment.yaml -n pypsa-eur
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: lockfiles
         path: |
@@ -75,12 +75,12 @@ jobs:
     steps:
     - uses: actions/checkout@v6
     - name: Download all artifacts
-      uses: actions/download-artifact@v6
+      uses: actions/download-artifact@v7
       with:
         name: lockfiles
 
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v7
+      uses: peter-evans/create-pull-request@v8
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         branch: update-locked-environment

.gitignore

@@ -5,9 +5,12 @@
 
 master
 
+.env
+
 .snakemake*
 .ipynb_checkpoints
 __pycache__
+*.egg-info
 *dconf
 gurobi.log
 .vscode
@@ -37,7 +40,6 @@ config/config.yaml
 config/scenarios.yaml
 config/config.private.yaml
 
-dconf
 /data/links_p_nom.csv
 /data/*totals.csv
 /data/biomass*
@@ -50,7 +52,6 @@ dconf
 /data/transport_data.csv
 /data/.nfs*
 /data/retro/*
-/data/bundle/nuts*
 data/gas_network/scigrid-gas/
 data/costs_*.csv
 !/data/custom_costs.csv

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
 # Run ruff to lint and format
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.13.3
+  rev: v0.14.10
   hooks:
     # Run the linter.
   - id: ruff
@@ -58,7 +58,7 @@ repos:
 
   # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v6.0.0
+  rev: v6.2.0
   hooks:
   - id: reuse-lint-file