Merge branch 'master' into feat/extend-tyndp-archive

Author: tgilon

.github/workflows/codeql.yaml

@@ -55,7 +55,7 @@ jobs:
         # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/push-images.yaml

@@ -14,7 +14,7 @@ jobs:
     name: dev-env
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         fetch-depth: 0
 

.github/workflows/release.yaml

@@ -16,7 +16,7 @@ jobs:
     outputs:
       prepared: ${{ steps.validate.outputs.prepared }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
 
     - name: Validate commit message
       id: validate
@@ -46,7 +46,7 @@ jobs:
         app-id: ${{ vars.PYPSA_BOT_ID }}
         private-key: ${{ secrets.PYPSA_BOT_PRIVATE_KEY }}
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         fetch-depth: 0
 
@@ -56,14 +56,14 @@ jobs:
         echo "Branch found: $branch"
         echo "BRANCH_NAME=$branch" >> $GITHUB_ENV
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         fetch-depth: 0
         ref: ${{ env.BRANCH_NAME }}
         token: ${{ steps.generate-token.outputs.token }}
 
     - name: Setup Pixi
-      uses: prefix-dev/setup-pixi@v0.9.6
+      uses: prefix-dev/setup-pixi@v0.10.0
       with:
         pixi-version: v0.68.1
         cache: true

.github/workflows/test.yaml

@@ -20,11 +20,11 @@ jobs:
     name: Validate Zenodo metadata
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
 
     - name: Cache Zenodo JSON schema
       id: cache-schema
-      uses: actions/cache@v5
+      uses: actions/cache@v6
       with:
         path: /tmp/zenodo_schema.json
         key: zenodo-schema-${{ hashFiles('.pre-commit-config.yaml') }}
@@ -48,10 +48,10 @@ jobs:
         shell: bash -l {0}
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
 
     - name: Setup Pixi
-      uses: prefix-dev/setup-pixi@v0.9.6
+      uses: prefix-dev/setup-pixi@v0.10.0
       with:
         pixi-version: v0.68.1
         cache: true
@@ -75,7 +75,7 @@ jobs:
         shell: bash -l {0}
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
 
     - uses: dorny/paths-filter@v4
       id: filter
@@ -108,7 +108,7 @@ jobs:
 
     - name: Setup Pixi
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-      uses: prefix-dev/setup-pixi@v0.9.6
+      uses: prefix-dev/setup-pixi@v0.10.0
       with:
         pixi-version: v0.68.1
         cache: true
@@ -122,13 +122,13 @@ jobs:
         echo "MONTH=$(date +'%Y%m')" >> $GITHUB_ENV # cutouts
         echo "VERSIONS_HASH=${{ hashFiles('data/versions.csv') }}" >> $GITHUB_ENV
 
-    - uses: actions/cache@v5
+    - uses: actions/cache@v6
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       with:
         path: data
         key: data-${{ env.WEEK }}-${{ env.VERSIONS_HASH }}
 
-    - uses: actions/cache@v5
+    - uses: actions/cache@v6
       if: steps.filter.outputs.src == 'true' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
       with:
         path: cutouts

.github/workflows/update-lockfile.yaml

@@ -9,10 +9,38 @@ on:
   workflow_dispatch:
 
 jobs:
+  check-diff:
+    runs-on: ubuntu-latest
+    outputs:
+      has_diff: ${{ steps.check_diff.outputs.has_diff }}
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        fetch-depth: 0
+
+    - name: Check pixi files diff
+      id: check_diff
+      run: |
+        if [[ "${{ github.ref }}" == "refs/heads/master" ]]; then
+          echo "has_diff=true" >> $GITHUB_OUTPUT
+          echo "Running on master branch - will proceed with update of lockfiles."
+          exit 0
+        fi
+
+        git fetch origin master
+        if git diff origin/master...HEAD -- pixi.toml pixi.lock | grep -q .; then
+          echo "has_diff=true" >> $GITHUB_OUTPUT
+          echo "Differences found compared to master - will proceed with update of lockfiles."
+        else
+          echo "has_diff=false" >> $GITHUB_OUTPUT
+          echo "No differences found compared to master - will skip update of lockfiles."
+        fi
   update-lockfiles:
+    needs: check-diff
     if: |
-      (github.event_name == 'schedule' && github.ref == 'refs/heads/master') ||
-      (github.event_name != 'schedule' && github.ref != 'refs/heads/master')
+      needs.check-diff.outputs.has_diff == 'true' &&
+      ((github.event_name == 'schedule' && github.ref == 'refs/heads/master') ||
+      (github.event_name != 'schedule' && github.ref != 'refs/heads/master'))
     name: Update lockfiles
     runs-on: ubuntu-latest
     permissions:
@@ -22,10 +50,10 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
 
     - name: Setup Pixi
-      uses: prefix-dev/setup-pixi@v0.9.6
+      uses: prefix-dev/setup-pixi@v0.10.0
       with:
         pixi-version: v0.68.1
 

.github/workflows/windows-installer.yaml

@@ -28,7 +28,7 @@ jobs:
     env:
       TAG: ${{ github.event.release.tag_name || inputs.release_tag }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
       with:
         ref: ${{ env.TAG || github.event.issue.pull_request && format('refs/pull/{0}/head', github.event.issue.number) }}
         fetch-depth: 0
@@ -46,7 +46,7 @@ jobs:
         git checkout master
 
     - name: Setup Pixi
-      uses: prefix-dev/setup-pixi@v0.9.6
+      uses: prefix-dev/setup-pixi@v0.10.0
       with:
         pixi-version: v0.59.0
         cache: true

doc/benchmarking.md

@@ -12,7 +12,7 @@ and Cost-Benefit Analysis (CBA) phases.
 * **Cost-Benefit Analysis** indicators are benchmarked for all calculated indicators and
   all projects against the official [TYNDP 2024 CBA results](https://tyndp2024.entsoe.eu/projects-map/transmission).
 
-Benchmarking results are published with every release on [Zenodo](https://doi.org/10.5281/zenodo.18608105) (currently only Scenario Building results).
+Benchmarking results continue to be published with every release on [Zenodo](https://doi.org/10.5281/zenodo.18608105) (currently only Scenario Building results).  View the [project website](https://open-tyndp.openenergytransition.org) for a detailed explanation of the results.
 
 !!! warning
     Open-TYNDP is under active development and is not yet feature-complete. The current

doc/index.md

@@ -17,10 +17,9 @@
 [![Discord](https://img.shields.io/discord/911692131440148490?logo=discord)](https://discord.gg/AnuJBk23FU)
 [![REUSE](https://api.reuse.software/badge/github.com/open-energy-transition/open-tyndp)](https://api.reuse.software/info/github.com/open-energy-transition/open-tyndp)
 
-|
 
 !!! warning
-    Open-TYNDP is under active development and is not yet feature-complete. The current [development status](#development-status) and the general [Limitations](limitations.md) are important to understand before using the model.
+    Open-TYNDP is under active development and is not yet feature-complete. As of v0.7, Open TYNDP was successfully benchmarked against TYNDP 2024 (see [results](https://open-tyndp.openenergytransition.org)). The current [development status](#development-status) and the general [Limitations](limitations.md) are important to understand before using the model.
 
 
 This repository introduces the open-source model of the Open-TYNDP research and innovation project, which is a collaboration between [Open Energy Transition (OET)](https://openenergytransition.org/) and the European Network of Transmission System Operators for Electricity (ENTSO-E).
@@ -67,12 +66,11 @@ Within this ecosystem, independent organisations can develop their own private r
 
 Each organisation in this open-source ecosystem remains fully independent of the shared foundation, maintaining its own governance structure and decision-making processes regarding its codebase. Organisations are also free to keep parts of their code and sensitive data completely private.
 
-|
 
 ## Development status {#development-status}
 
 !!! warning
-    Open-TYNDP is under active development and is not yet feature-complete. The current development status and general [limitations](limitations.md) are important to understand before using the model. The model includes partial data from the TYNDP 2024 cycle, and its benchmarking is ongoing. The github repository [issues](https://github.com/open-energy-transition/open-tyndp/issues) collects known topics we are working on (please feel free to help or make suggestions). The fact that this project relies on a soft-fork strategy implies that [upstream issues](https://github.com/PyPSA/PyPSA-Eur/issues) need to be addressed in the PyPSA-Eur repository. This [documentation](https://open-tyndp.readthedocs.io/) also remains work in progress.
+    Open-TYNDP is under active development and is not yet feature-complete. The current development status and general [limitations](limitations.md) are important to understand before using the model. Benchmarking against the TYNDP 2024 cycle is complete and the results can be viewed [online](https://open-tydnp.openenergytransition.org). The github repository [issues](https://github.com/open-energy-transition/open-tyndp/issues) collects known topics we are working on (please feel free to help or make suggestions). The fact that this project relies on a soft-fork strategy implies that [upstream issues](https://github.com/PyPSA/PyPSA-Eur/issues) need to be addressed in the PyPSA-Eur repository. This [documentation](https://open-tyndp.readthedocs.io/) also remains work in progress.
 
 ### Already implemented features
 
@@ -108,7 +106,6 @@ The back-casting of the 2024 TYNDP cycle involves developing new features based
 </tbody>
 </table>
 
-|
 
 ### Features in development
 
@@ -153,23 +150,20 @@ While multiple TYNDP features are already introduced to the Open-TYNDP model, th
 !!! note "See also"
     See also the [GitHub repository issues](https://github.com/open-energy-transition/open-tyndp/issues) for a comprehensive list of currently open issues.
 
-|
 
 ## Citing Open-TYNDP
 
 If you want to cite a specific Open-TYNDP version, since v0.6, each release is archived on Zenodo with a release-specific DOI:
 
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.19372053.svg)](https://doi.org/10.5281/zenodo.19372053)
 
-|
 
 Versions v0.5 and v0.5.1 are archived at [10.5281/zenodo.18494362](https://doi.org/10.5281/zenodo.18494362).
 
 If you use Open-TYNDP in your research, please cite it as shown on Zenodo or using the "Cite this repository" button in the sidebar of [Open-TYNDP](https://github.com/open-energy-transition/open-tyndp).
 
 This work builds upon [PyPSA-Eur](https://github.com/pypsa/pypsa-eur) and follows the methodology described in [ENTSO-E's and entsog's TYNDP 2024 Scenarios Methodology Report](https://2024.entsos-tyndp-scenarios.eu/scenarios-methodology-report/).
 
-|
 
 ## Background on PyPSA-Eur
 
@@ -325,4 +319,3 @@ If you want to cite a specific PyPSA-Eur version, each release of PyPSA-Eur is s
 
 The PyPSA-Eur workflow is continuously tested for Linux, macOS and Windows (WSL only).
 
-|

doc/release_notes.md

@@ -31,19 +31,32 @@
 
 **Bugfixes and Compatibility**
 
+* Rename bus for `t339` project (Tyrrhenian) from ITSI to ITVI ([#751](https://github.com/open-energy-transition/open-tyndp/pull/751)).
+
+* Fix CBA workflow to (a) release solver license after each successful rolling horizon optimization or after computing infeasibilities and (b) raise an error if rolling horizon fails when using HiGHS ([#756](https://github.com/open-energy-transition/open-tyndp/pull/756)).
+
 * Extend `tyndp-archive` support to integrate three new datasets (`desnz_electricity_consumption`, `jrc_energy_atlas`, `ons_lad`) and two new versions (`nitrogen_statistics`, `synthetic_electricity_demand`) ([#758](https://github.com/open-energy-transition/open-tyndp/pull/758)).
 
+
 **Documentation**
 
 * Update benchmarking documentation tables and figures for v0.7.1 ([#711](https://github.com/open-energy-transition/open-tyndp/pull/711)).
 
-* Migrate the Sphinx/RST-based documentation to MkDocs/Markdown, as a follow-up to the [upstream migration](https://github.com/PyPSA/pypsa-eur/pull/2162) ([754](https://github.com/open-energy-transition/open-tyndp/pull/754)).
+* Migrate the Sphinx/RST-based documentation to MkDocs/Markdown, as a follow-up to the [upstream migration](https://github.com/PyPSA/pypsa-eur/pull/2162) ([#754](https://github.com/open-energy-transition/open-tyndp/pull/754)).
 
 **Developers Note**
 
 * Change GitHub issue templates to comply with ISO security checks ([#714](https://github.com/open-energy-transition/open-tyndp/pull/714), [#730](https://github.com/open-energy-transition/open-tyndp/pull/730)).
 
-* Introduce SBOM/Grype vulnerability scanning workflow, as a follow-up to the [upstream addition](https://github.com/PyPSA/pypsa-eur/pull/2164) ([754](https://github.com/open-energy-transition/open-tyndp/pull/754)).
+* Introduce SBOM/Grype vulnerability scanning workflow, as a follow-up to the [upstream addition](https://github.com/PyPSA/pypsa-eur/pull/2164) ([#754](https://github.com/open-energy-transition/open-tyndp/pull/754)).
+
+* Ensure `inflow_t` is always defined in `attach_hydro`, resolving a pylint use-before-assignment issue ([#777](https://github.com/open-energy-transition/open-tyndp/pull/777)).
+
+* Add missing regex anchors with `re.fullmatch` to `create_zenodo_deposition_cli` utils script ([#778](https://github.com/open-energy-transition/open-tyndp/pull/778)).
+
+* Add validation of Zenodo deposition ID in `create_zenodo_deposition_cli` to close URL-manipulation finding ([#780](https://github.com/open-energy-transition/open-tyndp/pull/780)).
+
+* Prevent `create_zenodo_deposition_cli` from leaking secrets in logs under `--debug` by scoping debug logging to the local logger instead of the root logger ([#781](https://github.com/open-energy-transition/open-tyndp/pull/781)).
 
 
 ## Upcoming PyPSA-Eur Release
@@ -1011,7 +1024,7 @@
       full year(s) and then sliced to the subannual periods. This is to align
       better with the normalization process that uses annual EIA statistics.
 
-    - In [time_aggregation][], the resampling will now be applied separately
+    - In [build_snapshot_weightings][], the resampling will now be applied separately
       for each year covered in the snapshots. This prevents snapshots that
       overflow into the next year, which simplifies running non-contiguous
       periods.
@@ -1239,12 +1252,12 @@
 * Fail on solving status 'warning' because results are likely not valid.
   ([#1591](https://github.com/PyPSA/pypsa-eur/pull/1591))
 
-* Bugfix in [time_aggregation][]. The resampling produces a contiguous date
+* Bugfix in [build_snapshot_weightings][]. The resampling produces a contiguous date
   range. In case the original index was not contiguous, all rows with zero
   weight must be dropped (corresponding to time steps not included in the
   original snapshots). ([#1613](https://github.com/PyPSA/pypsa-eur/pull/1613))
 
-* Bugfix in [time_aggregation][]. Avoid that aggregated snapshot indices land
+* Bugfix in [build_snapshot_weightings][]. Avoid that aggregated snapshot indices land
   on February 29th in leap years when `enable: drop_leap_day: true`.
   ([#1613](https://github.com/PyPSA/pypsa-eur/pull/1613))
 
@@ -1508,7 +1521,7 @@
   [prepare_sector_network][] to correctly offset the corresponding oil
   emissions. ([#1410](https://github.com/PyPSA/pypsa-eur/pull/1410))
 
-* Add `{sector_opts}` wildcard to [time_aggregation][].
+* Add `{sector_opts}` wildcard to [build_snapshot_weightings][].
   ([#1307](https://github.com/PyPSA/pypsa-eur/pull/1307))
 
 * Resolved a problem where excluding certain countries from `countries`
@@ -2032,7 +2045,7 @@
   ([#1064](https://github.com/PyPSA/pypsa-eur/pull/1064))
 
 * Time aggregation for sector-coupled networks is now its own rule
-  [time_aggregation][]. Time aggregation is constant over planning horizons
+  [build_snapshot_weightings][]. Time aggregation is constant over planning horizons
   of the same network when using time step segmentation.
   ([#1065](https://github.com/PyPSA/pypsa-eur/pull/1065),
   [#1075](https://github.com/PyPSA/pypsa-eur/pull/1075))

doc/retrieve.md

@@ -14,7 +14,7 @@ The rules download data into subfolders in the `data/` directory, following the
 Which specific data version is retrieved can be controlled in the [data configuration](configuration.md#data_cf).
 
 For Open-TYNDP runs, most datasets can also be retrieved from a dedicated Google Cloud Storage
-bucket instead of their original sources. See [tyndp_archive](solving.md#tyndp_archive) in the SB documentation.
+bucket instead of their original sources. See [tyndp_archive](sb.md#tyndp_archive) in the SB documentation.
 
 Below some specific `retrieve_<dataset>` rules are documented.
 For more information on the datasets retrieved, see the [data sources](data_sources.md) and *Data inventory* section there in the documentation.