Skip to content

Commit 3bd68a6

Browse files
authored
Merge pull request open-energy-transition#96 from open-energy-transition/merge-upstream-260612
Merge changes from upstream
2 parents 981654c + 18774d5 commit 3bd68a6

169 files changed

Lines changed: 43083 additions & 40057 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/pull_request_template.md

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ Closes # (if applicable).
55

66
## Checklist
77

8-
- [ ] I tested my contribution locally and it works as intended.
9-
- [ ] Code and workflow changes are sufficiently documented.
10-
- [ ] Changed dependencies are added to `pixi.toml` (using `pixi add <dependency-name>`).
11-
- [ ] Changes in configuration options are added in `config/config.default.yaml`.
12-
- [ ] Changes in configuration options are documented in `doc/configtables/*.csv`.
8+
**Required:**
9+
- [ ] Changes are tested locally and behave as expected.
10+
- [ ] Code and workflow changes are documented.
11+
- [ ] A release note entry is added to `doc/release_notes.md`.
12+
13+
**If applicable:**
14+
- [ ] Changes in configuration options are reflected in `scripts/lib/validation`.
1315
- [ ] OET SPDX license header added to all touched files.
1416
- [ ] For new data sources or versions, [these instructions](https://pypsa-eur.readthedocs.io/en/latest/data_sources.html) have been followed.
15-
- [ ] A release note `doc/release_notes.rst` is added.
17+
- [ ] New rules are documented in the appropriate `doc/*.md` files.

.github/pypsa-bot.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,16 @@ tasks:
1515
- "doc/"
1616
- "*.yaml"
1717
- "*.yml"
18+
19+
app:
20+
trusted_users:
21+
- daniel-rdt
22+
runs:
23+
- name: "config.validator.yaml"
24+
configfile: "config/test/config.validator.yaml"
25+
triggers: [main, pr]
26+
cache:
27+
key: "validator-{YYYY}-{MM}"
28+
dirs: ["data"]
29+
import_networks:
30+
- "results/validation/networks/base_s_50___2050.nc"

.github/workflows/push-images.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
echo "hash_last_changed=$hash_last_changed" >> $GITHUB_ENV
2525
2626
- name: 'Login to GitHub Container Registry'
27-
uses: docker/login-action@v3
27+
uses: docker/login-action@v4
2828
with:
2929
registry: ghcr.io
3030
username: ${{ github.repository_owner }}

.github/workflows/release.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
steps:
4242
- name: Generate token for PyPSA Bot
4343
id: generate-token
44-
uses: actions/create-github-app-token@v2
44+
uses: actions/create-github-app-token@v3
4545
with:
4646
app-id: ${{ vars.PYPSA_BOT_ID }}
4747
private-key: ${{ secrets.PYPSA_BOT_PRIVATE_KEY }}
@@ -63,9 +63,9 @@ jobs:
6363
token: ${{ steps.generate-token.outputs.token }}
6464

6565
- name: Setup Pixi
66-
uses: prefix-dev/setup-pixi@v0.9.4
66+
uses: prefix-dev/setup-pixi@v0.9.6
6767
with:
68-
pixi-version: v0.59.0
68+
pixi-version: v0.68.1
6969
cache: true
7070
# Do not cache in branches
7171
cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: Security scan
2+
on:
3+
push:
4+
branches: [master]
5+
paths: [pixi.lock]
6+
pull_request:
7+
branches: [master]
8+
paths: [pixi.lock]
9+
schedule:
10+
- cron: "0 6 * * *"
11+
workflow_dispatch:
12+
13+
jobs:
14+
scan:
15+
name: SBOM and vulnerability scan
16+
runs-on: ubuntu-latest
17+
permissions:
18+
security-events: write
19+
contents: write # required by anchore/sbom-action dependency-snapshot upload
20+
actions: read # required by dawidd6/action-download-artifact on fork PRs
21+
defaults:
22+
run:
23+
shell: bash -l {0}
24+
steps:
25+
- uses: actions/checkout@v6
26+
- uses: prefix-dev/setup-pixi@v0.9.6
27+
with:
28+
pixi-version: v0.68.1
29+
30+
- name: Generate SBOM # (master only)
31+
if: github.ref == 'refs/heads/master'
32+
uses: anchore/sbom-action@v0.24.0
33+
with:
34+
path: ".pixi/envs/default"
35+
output-file: sbom.spdx.json
36+
dependency-snapshot: true
37+
38+
- name: Vulnerability Scan
39+
id: scan
40+
uses: anchore/scan-action@v7.4.0
41+
with:
42+
path: ".pixi/envs/default"
43+
fail-build: false
44+
severity-cutoff: high
45+
add-cpes-if-none: true
46+
output-format: sarif
47+
48+
# Point alerts to pixi.lock so githubs path scoped PR check registers it
49+
- name: Rewrite SARIF locations to pixi.lock
50+
run: |
51+
jq '.runs[].results[].locations[].physicalLocation.artifactLocation.uri = "pixi.lock"' \
52+
"${{ steps.scan.outputs.sarif }}" > rewritten.sarif
53+
mv rewritten.sarif "${{ steps.scan.outputs.sarif }}"
54+
55+
- name: Upload SARIF
56+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
57+
uses: github/codeql-action/upload-sarif@v4
58+
with:
59+
sarif_file: ${{ steps.scan.outputs.sarif }}
60+
category: pixi-vuln-scan
61+
62+
# Fork PRs cant upload SARIF (no access to token), so code scannings native diff
63+
# does not apply. This builds our own diff from a master-uploaded SARIF artifact instead.
64+
- name: Upload SARIF as artifact
65+
if: github.ref == 'refs/heads/master'
66+
uses: actions/upload-artifact@v7
67+
with:
68+
name: master-sarif
69+
path: ${{ steps.scan.outputs.sarif }}
70+
retention-days: 30
71+
72+
# Fork PRs cant upload SARIF to code scanning (no token write access), so
73+
# diff against the master baseline artifact here instead
74+
- name: Download master baseline (fork PRs only)
75+
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
76+
uses: dawidd6/action-download-artifact@v21
77+
with:
78+
workflow: security-scan.yaml
79+
name: master-sarif
80+
branch: master
81+
if_no_artifact_found: fail
82+
path: master-baseline
83+
84+
- name: Fail on new CVEs (fork PRs only)
85+
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
86+
env:
87+
SARIF_PATH: ${{ steps.scan.outputs.sarif }}
88+
run: |
89+
[ -s master-baseline/output ] || { echo "::error::baseline SARIF missing"; exit 2; }
90+
[ -s "$SARIF_PATH" ] || { echo "::error::PR SARIF missing"; exit 2; }
91+
new=$(comm -23 \
92+
<(jq -r '.runs[].results[] | select(.level == "error") | .ruleId' "$SARIF_PATH" | sort -u) \
93+
<(jq -r '.runs[].results[] | select(.level == "error") | .ruleId' master-baseline/output | sort -u))
94+
if [ -n "$new" ]; then
95+
echo "::error::New high+ CVEs vs master:"
96+
printf '%s\n' "$new"
97+
exit 1
98+
fi
99+
echo "No new high+ CVEs vs master."

.github/workflows/test.yaml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ jobs:
3030
- uses: actions/checkout@v6
3131

3232
- name: Setup Pixi
33-
uses: prefix-dev/setup-pixi@v0.9.4
33+
uses: prefix-dev/setup-pixi@v0.9.6
3434
with:
35-
pixi-version: v0.59.0
35+
pixi-version: v0.68.1
3636
cache: true
3737
cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
3838

@@ -56,7 +56,7 @@ jobs:
5656
steps:
5757
- uses: actions/checkout@v6
5858

59-
- uses: dorny/paths-filter@v3
59+
- uses: dorny/paths-filter@v4
6060
id: filter
6161
with:
6262
filters: |
@@ -87,9 +87,9 @@ jobs:
8787

8888
- name: Setup Pixi
8989
if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
90-
uses: prefix-dev/setup-pixi@v0.9.4
90+
uses: prefix-dev/setup-pixi@v0.9.6
9191
with:
92-
pixi-version: v0.59.0
92+
pixi-version: v0.68.1
9393
cache: true
9494
cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
9595

@@ -120,7 +120,7 @@ jobs:
120120
if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
121121
# check for undefined variables to reuse functions across scripts
122122
run: |
123-
pixi run pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*
123+
pixi run -e dev pylint --disable=all --enable=E0601,E0606 --output-format=parseable scripts/add_* scripts/prepare_* scripts/solve_*
124124
125125
- name: Run snakemake test workflows
126126
if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'

.github/workflows/update-lockfile.yaml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,24 @@ jobs:
99
if: ${{ github.ref == 'refs/heads/master' }}
1010
name: Update lockfiles
1111
runs-on: ubuntu-latest
12+
permissions:
13+
contents: write
1214
defaults:
1315
run:
1416
shell: bash -l {0}
1517
steps:
1618
- uses: actions/checkout@v6
1719

1820
- name: Setup Pixi
19-
uses: prefix-dev/setup-pixi@v0.9.4
21+
uses: prefix-dev/setup-pixi@v0.9.6
2022
with:
21-
pixi-version: v0.59.0
23+
pixi-version: v0.68.1
2224

2325
- name: Full resolve
2426
run: |
2527
rm pixi.lock
2628
pixi install --all
29+
git checkout pixi.toml
2730
2831
- name: Export conda environment files
2932
if: ${{ !vars.PYPSA_BOT_ID }}

.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ CLAUDE.md
116116
.claude/
117117
.github/copilot-instructions.md
118118
CLAUDE.local.md
119+
# generated DAGs for docs
120+
doc/img/dag_*.svg
121+
doc/img/workflow.svg
122+
doc/img/intro-workflow.svg
123+
119124
# pixi environments
120125
.pixi/*
121126
!.pixi/config.toml

.pre-commit-config.yaml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ repos:
1616
# Run ruff to lint and format
1717
- repo: https://github.com/astral-sh/ruff-pre-commit
1818
# Ruff version.
19-
rev: v0.14.10
19+
rev: v0.15.9
2020
hooks:
2121
# Run the linter.
2222
- id: ruff
@@ -26,7 +26,7 @@ repos:
2626

2727
# Find common spelling mistakes in comments and docstrings
2828
- repo: https://github.com/codespell-project/codespell
29-
rev: v2.4.1
29+
rev: v2.4.2
3030
hooks:
3131
- id: codespell
3232
args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor,pris,bund,ons'] # Ignore capital case words, e.g. country codes
@@ -43,15 +43,15 @@ repos:
4343

4444
# YAML formatting
4545
- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
46-
rev: v2.15.0
46+
rev: v2.16.0
4747
hooks:
4848
- id: pretty-format-yaml
4949
exclude: (\.lock\.yaml)$
5050
args: [--autofix, --indent, "2", --preserve-quotes]
5151

5252
# Format Snakemake rule / workflow files
5353
- repo: https://github.com/snakemake/snakefmt
54-
rev: v0.11.2
54+
rev: v1.0.0
5555
hooks:
5656
- id: snakefmt
5757

.readthedocs.yml

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,23 @@
44

55
version: 2
66

7-
sphinx:
8-
configuration: doc/conf.py
9-
7+
mkdocs:
8+
configuration: mkdocs.yml
109

1110
build:
1211
os: ubuntu-24.04
1312
tools:
1413
python: "3.12"
14+
apt_packages:
15+
- graphviz
1516
jobs:
1617
create_environment:
1718
- asdf plugin add pixi
1819
- asdf install pixi latest
1920
- asdf global pixi latest
2021
install:
21-
- pixi install -e doc --frozen
22+
- pixi install -e default -e doc --frozen
2223
build:
2324
html:
24-
- pixi run build-docs $READTHEDOCS_OUTPUT html
25+
- pixi run update-dags
26+
- pixi run build-docs $READTHEDOCS_OUTPUT

0 commit comments

Comments
 (0)