Merge changes from upstream

.github/ISSUE_TEMPLATE/1-bug-community.yaml

@@ -11,6 +11,8 @@ body:
 
       Please do not post usage questions here. Ask them on the [PyPSA/Open-TYNDP Discord channel](https://discord.com/channels/911692131440148490/1414977512089321564) or contact tyndp@openenergytransition.org.
 
+      Please write this issue description yourself. Any potentially verbose AI-generated content should be marked.
+
 - type: checkboxes
   id: version_checks
   attributes:

.github/ISSUE_TEMPLATE/2-feature-community.yaml

@@ -11,6 +11,8 @@ body:
 
       Please do not post usage questions here. Ask them on the [PyPSA/Open-TYNDP Discord channel](https://discord.com/channels/911692131440148490/1414977512089321564) or contact tyndp@openenergytransition.org.
 
+      Please write this issue description yourself. Any potentially verbose AI-generated content should be marked.
+
 - type: checkboxes
   id: affected_component
   attributes:

.github/ISSUE_TEMPLATE/3-bug-maintainer.yaml

@@ -11,6 +11,8 @@ body:
 
       Please do not post usage questions here. Ask them on the [PyPSA/Open-TYNDP Discord channel](https://discord.com/channels/911692131440148490/1414977512089321564) or contact tyndp@openenergytransition.org.
 
+      Please write this issue description yourself. Any potentially verbose AI-generated content should be marked.
+
 - type: checkboxes
   id: version_checks
   attributes:

.github/ISSUE_TEMPLATE/4-feature-maintainer-parent.yaml

@@ -12,6 +12,8 @@ body:
 
       This template is for high-level parent issues that will be broken down into sub-issues. Use the ["Feature - Sub Issue (Maintainer)"](https://github.com/open-energy-transition/open-tyndp/issues/new?template=5-feature-maintainer-sub.yaml) template for detailed feature requests. Security and risk assessment is captured at this sub-issue level.
 
+      Please write this issue description yourself. Any potentially verbose AI-generated content should be marked.
+
 - type: checkboxes
   id: affected_component
   attributes:

.github/ISSUE_TEMPLATE/5-feature-maintainer-sub.yaml

@@ -12,6 +12,8 @@ body:
 
       This template is for detailed feature requests. Use the ["Feature - Parent Issue (Maintainer)"](https://github.com/open-energy-transition/open-tyndp/issues/new?template=4-feature-maintainer-parent.yaml) template for high-level parent issues that will be broken down into sub-issues.
 
+      Please write this issue description yourself. Any potentially verbose AI-generated content should be marked.
+
 - type: input
   id: parent_issue
   attributes:

.github/ISSUE_TEMPLATE/6-spike-maintainer.yaml

@@ -12,6 +12,8 @@ body:
 
       Spikes are exploratory issues meant to gather information and explore concepts. They typically don't result in PRs but help inform future implementation decisions.
 
+      Please write this issue description yourself. Any potentially verbose AI-generated content should be marked.
+
 - type: input
   id: parent_issue
   attributes:

.github/pull_request_template.md

@@ -22,6 +22,7 @@ Closes # (if applicable).
 - [ ] Changes are tested locally and behave as expected.
 - [ ] Code and workflow changes are documented.
 - [ ] A release note entry is added to `doc/release_notes.md`.
+- [ ] The description is human-written and any AI-generated content is marked.
 
 **If applicable:**
 - [ ] Changes in configuration options are reflected in `scripts/lib/validation`.

.github/workflows/security-scan.yaml

@@ -22,8 +22,8 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-    - uses: actions/checkout@v6
-    - uses: prefix-dev/setup-pixi@v0.9.6
+    - uses: actions/checkout@v7
+    - uses: prefix-dev/setup-pixi@v0.10.0
       with:
         pixi-version: v0.68.1
 
@@ -45,12 +45,15 @@ jobs:
         add-cpes-if-none: true
         output-format: sarif
 
-    # Point alerts to pixi.lock so githubs path scoped PR check registers it
-    - name: Rewrite SARIF locations to pixi.lock
+    # Set uri=pixi.lock for a readable location, and fingerprint each alert by
+    # CVE id so code scanning matches against the master baseline by CVE, not line.
+    - name: Normalize SARIF (pixi.lock location + CVE fingerprint)
       run: |
-        jq '.runs[].results[].locations[].physicalLocation.artifactLocation.uri = "pixi.lock"' \
-          "${{ steps.scan.outputs.sarif }}" > rewritten.sarif
-        mv rewritten.sarif "${{ steps.scan.outputs.sarif }}"
+        jq '.runs[].results[] |= (
+              .locations[].physicalLocation.artifactLocation.uri = "pixi.lock"
+              | .partialFingerprints = {"grypeCve/v1": .ruleId}
+            )' "${{ steps.scan.outputs.sarif }}" > normalized.sarif
+        mv normalized.sarif "${{ steps.scan.outputs.sarif }}"
 
     - name: Upload SARIF
       if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository

Snakefile

@@ -60,16 +60,19 @@ for scenario_name, scenario_overrides in scenarios.items():
         ) from e
 
 RDIR = get_rdir(run)
+PROJ_DIR = Path(workflow.snakefile).parent
+
 shadow_config = get_shadow(run)
 
 shared_resources = run["shared_resources"]["policy"]
 exclude_from_shared = run["shared_resources"]["exclude"]
 logs = path_provider("logs/", RDIR, shared_resources, exclude_from_shared)
 benchmarks = path_provider("benchmarks/", RDIR, shared_resources, exclude_from_shared)
 resources = path_provider("resources/", RDIR, shared_resources, exclude_from_shared)
-scripts = script_path_provider(Path(workflow.snakefile).parent)
+scripts = script_path_provider(PROJ_DIR)
 
 RESULTS = "results/" + RDIR
+workflow.default_target = config["run"]["default_target_rule"]
 
 
 localrules:
@@ -207,7 +210,6 @@ def input_all_tyndp(w):
 
 
 rule all:
-    default_target: True
     input:
         input_all_tyndp,
         expand(RESULTS + "graphs/costs.pdf", run=config["run"]["name"]),
@@ -396,6 +398,8 @@ rule rulegraph:
         pdf=resources("dag_rulegraph.pdf"),
         png=resources("dag_rulegraph.png"),
         svg=resources("dag_rulegraph.svg"),
+    params:
+        default_target=workflow.default_target,  # track default target to ensure rule is re-triggered
     message:
         "Creating RULEGRAPH dag in multiple formats using the final configuration."
     shell:
@@ -433,6 +437,8 @@ rule filegraph:
         pdf=resources("dag_filegraph.pdf"),
         png=resources("dag_filegraph.png"),
         svg=resources("dag_filegraph.svg"),
+    params:
+        default_target=workflow.default_target,  # track default target to ensure rule is re-triggered
     message:
         "Creating FILEGRAPH dag in multiple formats using the final configuration."
     shell:

config/config.default.yaml

@@ -33,6 +33,7 @@ run:
     policy: false
     exclude: []
   use_shadow_directory: false
+  default_target_rule: all
 
 # docs in https://open-tyndp.readthedocs.io/en/latest/configuration.html#foresight
 foresight: overnight
@@ -1311,6 +1312,13 @@ solving:
       primal_feasibility_tolerance: 1.0e-05
       dual_feasibility_tolerance: 1.0e-05
       random_seed: 123
+    "highs-hipo":
+      solver: hipo
+      parallel: 'on'
+      primal_feasibility_tolerance: 1.0e-05
+      dual_feasibility_tolerance: 1.0e-05
+      random_seed: 123
+      run_crossover: 'off'
     "gurobi-default":
       threads: 32
       method: 2
@@ -1394,6 +1402,8 @@ solving:
 
 # docs in https://open-tyndp.readthedocs.io/en/latest/configuration.html#data
 data:
+  version_files:
+  - data/versions.csv
   hotmaps_industrial_sites:
     source: archive
     version: latest

config/plotting.default.yaml

@@ -686,6 +686,7 @@ plotting:
     heat demand: '#cc1f1f'
     rural heat: '#ff5c5c'
     rural heat dsm: '#ff5c5c'
+    heat dsm: '#ff5c5c'
     residential rural heat: '#ff7c7c'
     services rural heat: '#ff9c9c'
     central heat: '#cc1f1f'

config/schema.default.json

@@ -1068,6 +1068,14 @@
     "DataConfig": {
       "description": "Configuration for `data` settings.",
       "properties": {
+        "version_files": {
+          "description": "\n        List of paths to version files.\n        If multiple paths are provided, they will be merged with priority given to the later paths in the list.\n        This allows for overriding default versions with custom versions, without modifying the default version file.\n\n        All filepaths must be relative to the project root or as an absolute path and point to one of:\n        - a CSV file.\n        - a YAML file with a list of version entries, which will be converted internally to CSV format.\n          This allows for easier editing of versions with comments and multi-line notes.\n          This is equivalent to calling `df.to_dict(orient='records')` on a dataframe with the same structure as the CSV version files, and then dumping to YAML.\n\n        The CSV columns / YAML fields must be:\n        - `dataset`: The name of the dataset (e.g., \"hotmaps_industrial_sites\", \"enspreso_biomass\", etc.).\n        - `version`: The version of the dataset to use (e.g., \"latest\", \"v1.0\", etc.).\n        - `source`: The source of the dataset (e.g., \"archive\", \"primary\", \"build\").\n          \"archive\" retrieves data from an organisation data bucket (e.g. `data.pypsa.org`).\n          \"primary\" retrieves data from the primary source (e.g. Eurostat, OSM, etc.).\n          \"build\" retrieves data from the result of a build script within the PyPSA-Eur workflow itself, rather than a remote source.\n        - `tags`: space separated tags for the dataset, which will become boolean columns when loaded into the workflow.\n          \"supported\" must be a tag on a version if it is supported by the workflow.\n          \"latest\" must be a tag on the version of each `source` that should be used when `version` is set to \"latest\".\n        - `added`: The date when the dataset version was added (e.g., \"2024-01-01\").\n        - `note`: [Optional] notes about the dataset version.\n        - `url`: URL to the dataset version. Optional if data `source` is \"build\", otherwise required.\n        ",
+          "items": {
+            "format": "file-path",
+            "type": "string"
+          },
+          "type": "array"
+        },
         "hotmaps_industrial_sites": {
           "description": "Configuration for a single data source.",
           "properties": {
@@ -4703,6 +4711,18 @@
             true
           ],
           "type": "boolean"
+        },
+        "default_target_rule": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": "all",
+          "description": "Default target rule for snakemake. This is the default rule that will be executed if no other rule is specified. If set to ``None``, snakemake will use its default behavior and execute the first rule in the Snakefile. NOTE: This value will be overwritten by any reference to `default_target` made in a specific rule."
         }
       }
     },
@@ -9439,6 +9459,18 @@
             true
           ],
           "type": "boolean"
+        },
+        "default_target_rule": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": "all",
+          "description": "Default target rule for snakemake. This is the default rule that will be executed if no other rule is specified. If set to ``None``, snakemake will use its default behavior and execute the first rule in the Snakefile. NOTE: This value will be overwritten by any reference to `default_target` made in a specific rule."
         }
       }
     },
@@ -14180,6 +14212,14 @@
     "data": {
       "description": "Configuration for `data` settings.",
       "properties": {
+        "version_files": {
+          "description": "\n        List of paths to version files.\n        If multiple paths are provided, they will be merged with priority given to the later paths in the list.\n        This allows for overriding default versions with custom versions, without modifying the default version file.\n\n        All filepaths must be relative to the project root or as an absolute path and point to one of:\n        - a CSV file.\n        - a YAML file with a list of version entries, which will be converted internally to CSV format.\n          This allows for easier editing of versions with comments and multi-line notes.\n          This is equivalent to calling `df.to_dict(orient='records')` on a dataframe with the same structure as the CSV version files, and then dumping to YAML.\n\n        The CSV columns / YAML fields must be:\n        - `dataset`: The name of the dataset (e.g., \"hotmaps_industrial_sites\", \"enspreso_biomass\", etc.).\n        - `version`: The version of the dataset to use (e.g., \"latest\", \"v1.0\", etc.).\n        - `source`: The source of the dataset (e.g., \"archive\", \"primary\", \"build\").\n          \"archive\" retrieves data from an organisation data bucket (e.g. `data.pypsa.org`).\n          \"primary\" retrieves data from the primary source (e.g. Eurostat, OSM, etc.).\n          \"build\" retrieves data from the result of a build script within the PyPSA-Eur workflow itself, rather than a remote source.\n        - `tags`: space separated tags for the dataset, which will become boolean columns when loaded into the workflow.\n          \"supported\" must be a tag on a version if it is supported by the workflow.\n          \"latest\" must be a tag on the version of each `source` that should be used when `version` is set to \"latest\".\n        - `added`: The date when the dataset version was added (e.g., \"2024-01-01\").\n        - `note`: [Optional] notes about the dataset version.\n        - `url`: URL to the dataset version. Optional if data `source` is \"build\", otherwise required.\n        ",
+          "items": {
+            "format": "file-path",
+            "type": "string"
+          },
+          "type": "array"
+        },
         "hotmaps_industrial_sites": {
           "description": "Configuration for a single data source.",
           "properties": {

doc/configuration.md

@@ -5,7 +5,6 @@
 
 <a id="config"></a>
 
-# 
 # Configuration
 
 PyPSA-Eur has several configuration options which are documented in this section.
@@ -571,7 +570,7 @@ Only used for sector-coupling studies.
 ??? note "Details"
 
     Configuration for `sector` settings.
-    
+
     | Property | Type | Default | Description |
     |----------|------|---------|-------------|
     | `transport` | boolean | `true` | Flag to include transport sector. |
@@ -754,7 +753,7 @@ Only used for sector-coupling studies.
     |   `limit` | number |  | Maximum allowed renewable energy imports (TWh). |
     |   `limit_sense` | string | `<=` | Sense of the limit. |
     |   `price` | dict (str -> number) |  | Price for importing renewable energy of carrier. |
-    
+
 
 **YAML Syntax**
 
@@ -886,10 +885,11 @@ Datasets that are mirrored to the Open-TYNDP data store on Google Cloud Storage
 (see [tyndp_archive](solving.md#tyndp_archive)).
 See the `data/versions.csv` file for all available datasets and their sources/versions that are supported.
 
-??? note "Details"
+In your own project, you can define additional version files to modify the contents of `data/versions.csv`.
+See the [data versioning documentation](data_sources.md#managing_data_versions) for more details.
+
+??? note "Configuration for `data` settings."
 
-    Configuration for `data` settings.
-
     | Property | Type | Default | Description |
     |----------|------|---------|-------------|
     | `hotmaps_industrial_sites` | any |  | Configuration for a single data source. |
@@ -1069,7 +1069,7 @@ See the `data/versions.csv` file for all available datasets and their sources/ve
     | `bidding_zones_entsoepy` | any |  | Configuration for a single data source. |
     |   `source` | enum (`archive`, `primary`, `build`) | `archive` | Source of the data. 'archive' retrieves pre-built data, 'primary' retrieves from primary source. |
     |   `version` | string | `latest` | Version of the data to use. Uses the specific 'version' for the selected 'source' or the dataset tagged 'latest' for this source. |
-    
+
 
 **YAML Syntax**
 

doc/contributing.md

@@ -52,3 +52,20 @@ We strive to keep documentation useful and up to date for all PyPSA users. If yo
 3. Compile your changes by running `pixi run build-docs site` from the project root.
    You can also preview live with `mkdocs serve`. HTML files to review can be found under `site/`.
 4. Contribute your documentation in a pull request ([here is a guide](https://help.github.com/en/articles/about-pull-requests)).
+
+
+## AI-based Contributions
+
+To handle the influx of AI-assisted contributions and protect maintainer time, we have established some basic principles.
+
+We do not impose general restrictions on the use of AI for contributions.
+But we require that all contributions, whether AI-assisted or not, adhere to the same standards of quality, relevance, and maintainability.
+This boils down to:
+
+1. PR and issue descriptions should read like you wrote them. Keep them concise and human. Any potentially verbose AI-generated content should be marked, e.g. in a collapsed `<details>` block. Please don't mix both.
+2. Take responsibility for the content and quality of your contribution, even if you used AI.
+3. Keep the code simple. AI tools often produce verbose or over-engineered code. Review and trim it back before submitting.
+4. Before you submit large contributions, coordinate with the maintainers to ensure they align with the project's goals and roadmap.
+5. Minimize the review burden on us by keeping your contributions focused, don't solve multiple tangential issues in one PR.
+
+We may close AI-based contributions without further review if we consider them to bind up excessive maintainer time.