Merge changes from upstream (incl. PyPSA-Eur v2026.02.0)

.github/pull_request_template.md

@@ -17,17 +17,18 @@ Closes # (if applicable).
 
 ## Checklist
 
-- [ ] Security scan(s) show no high-severity bugs, critical vulnerabilities, or exposed secrets.
-- [ ] I tested my contribution locally and it works as intended.
-- [ ] Code and workflow changes are sufficiently documented.
-- [ ] Changed dependencies are added to `pixi.toml` (using `pixi add <dependency-name>`).
-- [ ] Changes in configuration options are added in `config/config.default.yaml`.
-- [ ] Changes in configuration options are documented in `doc/configtables/*.csv`.
-- [ ] Changes in configuration options are added in `config/test/*.yaml`.
-- [ ] The multiple weather/climate years test is passing locally (using `pixi run -e open-tyndp tyndp-cyears-test`).
-- [ ] Open-TYNDP SPDX license header added to all touched files.
+**Required:**
+- [ ] Security scans show no high-severity bugs, critical vulnerabilities, or exposed secrets.
+- [ ] Changes are tested locally and behave as expected.
+- [ ] Code and workflow changes are documented.
+- [ ] A release note entry is added to `doc/release_notes.md`.
+
+**If applicable:**
+- [ ] Changes in configuration options are reflected in `scripts/lib/validation`.
+- [ ] Changes in configuration options are added to `config/test/*.yaml`.
+- [ ] Multiple climate years test passes locally (`pixi run -e open-tyndp tyndp-cyears-test`).
 - [ ] For new data sources or versions, [these instructions](https://open-tyndp.readthedocs.io/en/latest/data_sources.html) have been followed.
-- [ ] New rules are documented in the appropriate `doc/*.rst` files.
-- [ ] A release note `doc/release_notes.rst` is added.
-- [ ] Major features are documented with up-to-date information in `doc/index.rst`.
-- [ ] Module docstrings added to new Python scripts.
+- [ ] Open-TYNDP SPDX license header is added to all touched files.
+- [ ] Module docstrings are added to new Python scripts.
+- [ ] New rules are documented in the appropriate `doc/*.md` files.
+- [ ] Major features are documented in `doc/index.md`.

.github/pypsa-bot.yml

@@ -0,0 +1,30 @@
+tasks:
+  sync-locks:
+    triggers:
+    - "pixi.toml"
+    - "pixi.lock"
+    stage:
+    - "pixi.lock"
+    - "envs/"
+
+  generate-config:
+    triggers:
+    - "scripts/lib/validation/config/**"
+    stage:
+    - "config/"
+    - "doc/"
+    - "*.yaml"
+    - "*.yml"
+
+app:
+  trusted_users:
+  - daniel-rdt
+  runs:
+  - name: "config.validator.yaml"
+    configfile: "config/test/config.validator.yaml"
+    triggers: [main, pr]
+    cache:
+      key: "validator-{YYYY}-{MM}"
+      dirs: ["data"]
+    import_networks:
+    - "results/validation/networks/base_s_50___2050.nc"

.github/workflows/push-images.yaml

@@ -18,9 +18,9 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: 'Get last commit that changed pixi env'
+    - name: 'Get last commit that changed pixi env or Dockerfile'
       run: |
-        hash_last_changed=$(git log -1 --pretty=format:%H -- pixi.toml pixi.lock)
+        hash_last_changed=$(git log -1 --pretty=format:%H -- pixi.toml pixi.lock docker/dev-env/Dockerfile)
         echo "hash_last_changed=$hash_last_changed" >> $GITHUB_ENV
 
     - name: 'Login to GitHub Container Registry'

.github/workflows/release.yaml

@@ -65,7 +65,7 @@ jobs:
     - name: Setup Pixi
       uses: prefix-dev/setup-pixi@v0.9.6
       with:
-        pixi-version: v0.59.0
+        pixi-version: v0.68.1
         cache: true
         # Do not cache in branches
         cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}

.github/workflows/security-scan.yaml

@@ -0,0 +1,99 @@
+name: Security scan
+on:
+  push:
+    branches: [master]
+    paths: [pixi.lock]
+  pull_request:
+    branches: [master]
+    paths: [pixi.lock]
+  schedule:
+  - cron: "0 6 * * *"
+  workflow_dispatch:
+
+jobs:
+  scan:
+    name: SBOM and vulnerability scan
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      contents: write   # required by anchore/sbom-action dependency-snapshot upload
+      actions: read     # required by dawidd6/action-download-artifact on fork PRs
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+    - uses: actions/checkout@v6
+    - uses: prefix-dev/setup-pixi@v0.9.6
+      with:
+        pixi-version: v0.68.1
+
+    - name: Generate SBOM # (master only)
+      if: github.ref == 'refs/heads/master'
+      uses: anchore/sbom-action@v0.24.0
+      with:
+        path: ".pixi/envs/default"
+        output-file: sbom.spdx.json
+        dependency-snapshot: true
+
+    - name: Vulnerability Scan
+      id: scan
+      uses: anchore/scan-action@v7.4.0
+      with:
+        path: ".pixi/envs/default"
+        fail-build: false
+        severity-cutoff: high
+        add-cpes-if-none: true
+        output-format: sarif
+
+    # Point alerts to pixi.lock so githubs path scoped PR check registers it
+    - name: Rewrite SARIF locations to pixi.lock
+      run: |
+        jq '.runs[].results[].locations[].physicalLocation.artifactLocation.uri = "pixi.lock"' \
+          "${{ steps.scan.outputs.sarif }}" > rewritten.sarif
+        mv rewritten.sarif "${{ steps.scan.outputs.sarif }}"
+
+    - name: Upload SARIF
+      if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+      uses: github/codeql-action/upload-sarif@v4
+      with:
+        sarif_file: ${{ steps.scan.outputs.sarif }}
+        category: pixi-vuln-scan
+
+    # Fork PRs cant upload SARIF (no access to token), so code scannings native diff
+    # does not apply. This builds our own diff from a master-uploaded SARIF artifact instead.
+    - name: Upload SARIF as artifact
+      if: github.ref == 'refs/heads/master'
+      uses: actions/upload-artifact@v7
+      with:
+        name: master-sarif
+        path: ${{ steps.scan.outputs.sarif }}
+        retention-days: 30
+
+    # Fork PRs cant upload SARIF to code scanning (no token write access), so
+    # diff against the master baseline artifact here instead
+    - name: Download master baseline (fork PRs only)
+      if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
+      uses: dawidd6/action-download-artifact@v21
+      with:
+        workflow: security-scan.yaml
+        name: master-sarif
+        branch: master
+        if_no_artifact_found: fail
+        path: master-baseline
+
+    - name: Fail on new CVEs (fork PRs only)
+      if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
+      env:
+        SARIF_PATH: ${{ steps.scan.outputs.sarif }}
+      run: |
+        [ -s master-baseline/output ] || { echo "::error::baseline SARIF missing"; exit 2; }
+        [ -s "$SARIF_PATH" ]          || { echo "::error::PR SARIF missing"; exit 2; }
+        new=$(comm -23 \
+          <(jq -r '.runs[].results[] | select(.level == "error") | .ruleId' "$SARIF_PATH"          | sort -u) \
+          <(jq -r '.runs[].results[] | select(.level == "error") | .ruleId' master-baseline/output | sort -u))
+        if [ -n "$new" ]; then
+          echo "::error::New high+ CVEs vs master:"
+          printf '%s\n' "$new"
+          exit 1
+        fi
+        echo "No new high+ CVEs vs master."

.github/workflows/test.yaml

@@ -53,7 +53,7 @@ jobs:
     - name: Setup Pixi
       uses: prefix-dev/setup-pixi@v0.9.6
       with:
-        pixi-version: v0.59.0
+        pixi-version: v0.68.1
         cache: true
         cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
 
@@ -103,39 +103,39 @@ jobs:
         df -h
 
     - name: Skip - no source changes
-      if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule'
+      if: steps.filter.outputs.src != 'true' && github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'
       run: echo "Skipping tests because no source code changes detected"
 
     - name: Setup Pixi
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       uses: prefix-dev/setup-pixi@v0.9.6
       with:
-        pixi-version: v0.59.0
+        pixi-version: v0.68.1
         cache: true
         cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
         environments: open-tyndp
 
     - name: Setup cache keys
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       run: |
         echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data
         echo "MONTH=$(date +'%Y%m')" >> $GITHUB_ENV # cutouts
         echo "VERSIONS_HASH=${{ hashFiles('data/versions.csv') }}" >> $GITHUB_ENV
 
     - uses: actions/cache@v5
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       with:
         path: data
         key: data-${{ env.WEEK }}-${{ env.VERSIONS_HASH }}
 
     - uses: actions/cache@v5
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       with:
         path: cutouts
         key: cutouts-${{ env.MONTH }}
 
     - name: Restore git-tracked files in data/
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       run: git checkout HEAD -- data/
 
     - name: Restore git-tracked files in data/
@@ -146,26 +146,26 @@ jobs:
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
       # check for undefined variables to reuse functions across scripts
       run: |
-        pixi run pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*
+        pixi run -e dev pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*
 
     - name: Run snakemake test workflows (sb)
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
         pixi run -e open-tyndp tyndp-sb-test
 
     - name: Run snakemake test workflows (cba)
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"
       run: |
         pixi run -e open-tyndp tyndp-cba-test
 
     - name: Run snakemake test workflows (upstream)
-      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule'
+      if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       env:
         SNAKEMAKE_STORAGE_CACHED_HTTP_CACHE: ""
         SNAKEMAKE_STORAGE_CACHED_HTTP_SKIP_REMOTE_CHECKS: "1"

.github/workflows/update-lockfile.yaml

@@ -8,48 +8,16 @@ on:
   - cron: "0 8 1,16 * *" # Bi-weekly
   workflow_dispatch:
 
-permissions:
-  contents: write
-  pull-requests: write
-
 jobs:
-  check-diff:
-    runs-on: ubuntu-latest
-    outputs:
-      has_diff: ${{ steps.check_diff.outputs.has_diff }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        fetch-depth: 0
-
-    - name: Check pixi files diff
-      id: check_diff
-      run: |
-        if [[ "${{ github.ref }}" == "refs/heads/master" ]]; then
-          echo "has_diff=true" >> $GITHUB_OUTPUT
-          echo "Running on master branch - will proceed with update of lockfiles."
-          exit 0
-        fi
-
-        git fetch origin master
-        if git diff origin/master...HEAD -- pixi.toml pixi.lock | grep -q .; then
-          echo "has_diff=true" >> $GITHUB_OUTPUT
-          echo "Differences found compared to master - will proceed with update of lockfiles."
-        else
-          echo "has_diff=false" >> $GITHUB_OUTPUT
-          echo "No differences found compared to master - will skip update of lockfiles."
-        fi
-      env:
-        GH_TOKEN: ${{ github.token }}
-
-  update-locked-environment:
+  update-lockfiles:
     if: |
-      needs.check-diff.outputs.has_diff == 'true' &&
-      (github.event_name == 'schedule' && github.ref == 'refs/heads/master' || 
-       github.event_name != 'schedule' && github.ref != 'refs/heads/master')
-    needs: check-diff
+      (github.event_name == 'schedule' && github.ref == 'refs/heads/master') ||
+      (github.event_name != 'schedule' && github.ref != 'refs/heads/master')
     name: Update lockfiles
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
     defaults:
       run:
         shell: bash -l {0}
@@ -59,65 +27,26 @@ jobs:
     - name: Setup Pixi
       uses: prefix-dev/setup-pixi@v0.9.6
       with:
-        pixi-version: v0.59.0
-        run-install: false
+        pixi-version: v0.68.1
 
-    - name: Full resolve (exclude packages newer than 7 days)
-      # Exclude recently published packages to avoid pulling in broken or
-      # yanked releases before upstream has had time to react
+    - name: Full resolve
       run: |
-        EXCLUDE_DATE=$(date -u -d '7 days ago' +%Y-%m-%dT00:00:00Z)
-        sed -i "/^\[workspace\]/a exclude-newer = \"$EXCLUDE_DATE\"" pixi.toml
         rm pixi.lock
         pixi install --all
         git checkout pixi.toml
 
-    - name: Create new conda lock files
-      run: |
-        pixi workspace export conda-explicit-spec -e default envs
-        for f in envs/*_conda_spec*; do mv "$f" "${f/_conda_spec/.pin}"; done
-
-    - name: Create new conda environment file
-      run: |
-        pixi workspace export conda-environment -e default envs/environment.yaml -n pypsa-eur
-
-    - name: Upload artifacts
-      uses: actions/upload-artifact@v7
-      with:
-        name: lockfiles
-        path: |
-          pixi.lock
-          envs/
-
-  create-commit:
-    if: ${{ github.ref != 'refs/heads/master' }}
-    needs: update-locked-environment
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v6
-
-    - name: Download all artifacts
-      uses: actions/download-artifact@v8
-      with:
-        name: lockfiles
+    - name: Export conda environment files
+      if: ${{ !vars.PYPSA_BOT_ID }}
+      run: pixi run sync-locks
 
     - name: Create Commit
+      if: ${{ github.ref != 'refs/heads/master' }}
       uses: stefanzweifel/git-auto-commit-action@v7
       with:
         commit_message: "[github-actions.ci] Update locked envs"
 
-  create-pull-request:
-    if: ${{ github.ref == 'refs/heads/master' }}
-    needs: update-locked-environment
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v6
-    - name: Download all artifacts
-      uses: actions/download-artifact@v8
-      with:
-        name: lockfiles
-
     - name: Create Pull Request
+      if: ${{ github.ref == 'refs/heads/master' }}
       uses: peter-evans/create-pull-request@v8
       with:
         token: ${{ secrets.GITHUB_TOKEN }}