Revert "Merge changes from upstream incl. data layer (#363)"

This reverts commit 0a5e64c.

Author: tgilon

.github/pull_request_template.md

@@ -17,14 +17,16 @@ Closes # (if applicable).
 
 ## Checklist
 
+<!-- Remove what doesn't apply. -->
+
 - [ ] I tested my contribution locally and it works as intended.
 - [ ] Code and workflow changes are sufficiently documented.
 - [ ] Changed dependencies are added to `pixi.toml` (using `pixi add <dependency-name>`).
 - [ ] Changes in configuration options are added in `config/config.default.yaml`.
 - [ ] Changes in configuration options are documented in `doc/configtables/*.csv`.
 - [ ] Changes in configuration options are added in `config/test/*.yaml`.
 - [ ] Open-TYNDP SPDX license header added to all touched files.
-- [ ] For new data sources or versions, [these instructions](https://pypsa-eur.readthedocs.io/en/latest/data_sources.html) have been followed.
+- [ ] Sources of newly added data are documented in `doc/data_sources.rst`.
 - [ ] New rules are documented in the appropriate `doc/*.rst` files.
 - [ ] A release note `doc/release_notes.rst` is added.
 - [ ] Major features are documented with up-to-date information in `README` and `doc/index.rst`.

.github/workflows/test.yaml

@@ -30,23 +30,6 @@ jobs:
         shell: bash -l {0}
 
     steps:
-    - uses: actions/checkout@v6
-
-    - uses: dorny/paths-filter@v3
-      id: filter
-      with:
-        filters: |
-          src:
-            - 'scripts/**'
-            - 'rules/**'
-            - 'data/**'
-            - 'Snakefile'
-            - 'config/**'
-            - 'test/**'
-            - 'pixi.toml'
-            - 'pixi.lock'
-            - '.github/workflows/test.yaml'
-
     - name: Free up disk space
       run: |
         echo "Initial disk space"
@@ -57,12 +40,10 @@ jobs:
         sudo docker builder prune -a --force
         echo "Final disk space"
         df -h
-    - name: Skip - no source changes
-      if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule'
-      run: echo "Skipping tests because no source code changes detected"
+
+    - uses: actions/checkout@v6
 
     - name: Setup Pixi
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       uses: prefix-dev/setup-pixi@v0.9.3
       with:
         pixi-version: v0.59.0
@@ -72,60 +53,49 @@ jobs:
         environments: open-tyndp
 
     - name: Setup cache keys
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: |
         echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts
 
-    - uses: actions/cache@v5
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+    - uses: actions/cache@v4
       with:
         path: |
           data
           cutouts
         key: data-cutouts-${{ env.WEEK }}
 
-    - name: Restore git-tracked files in data/
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
-      run: git checkout HEAD -- data/
-
     - name: Run pylint check on scripts
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       # check for undefined variables to reuse functions across scripts
       run: |
         pixi run pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*
 
     - name: Run snakemake test workflows (sb)
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
         pixi run -e open-tyndp tyndp-sb-test
 
     - name: Run snakemake test workflows (cba)
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
         pixi run -e open-tyndp tyndp-cba-test
 
     - name: Run snakemake test workflows (upstream)
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
         pixi run -e open-tyndp integration-tests
 
     - name: Run unit tests
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       run: |
         pixi run -e open-tyndp unit-tests
 
     - name: Upload artifacts
       if: always()
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v5
       with:
         name: results-${{ matrix.os }}
         path: |

.github/workflows/update-lockfile.yaml

@@ -76,7 +76,7 @@ jobs:
         pixi workspace export conda-environment -e default envs/environment.yaml -n pypsa-eur
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v5
       with:
         name: lockfiles
         path: |
@@ -107,12 +107,12 @@ jobs:
     steps:
     - uses: actions/checkout@v6
     - name: Download all artifacts
-      uses: actions/download-artifact@v7
+      uses: actions/download-artifact@v6
       with:
         name: lockfiles
 
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v8
+      uses: peter-evans/create-pull-request@v7
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
         branch: update-locked-environment

.gitignore

@@ -5,12 +5,9 @@
 
 master
 
-.env
-
 .snakemake*
 .ipynb_checkpoints
 __pycache__
-*.egg-info
 *dconf
 gurobi.log
 .vscode
@@ -40,6 +37,7 @@ config/config.yaml
 config/scenarios.yaml
 config/config.private.yaml
 
+dconf
 /data/links_p_nom.csv
 /data/*totals.csv
 /data/biomass*
@@ -52,6 +50,7 @@ config/config.private.yaml
 /data/transport_data.csv
 /data/.nfs*
 /data/retro/*
+/data/bundle/nuts*
 data/gas_network/scigrid-gas/
 data/costs_*.csv
 !/data/custom_costs.csv

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
 # Run ruff to lint and format
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.14.10
+  rev: v0.13.3
   hooks:
     # Run the linter.
   - id: ruff
@@ -58,7 +58,7 @@ repos:
 
   # Check for FSFE REUSE compliance (licensing)
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v6.2.0
+  rev: v6.0.0
   hooks:
   - id: reuse-lint-file