Get test matrix from hatch (#390)

* Get test matrix from hatch

* Fix test command

* Fix test command

* Create env

* fix create env

* Format

* install python

* Fix jq

* try officiall spelling

* try  uv python-version

* try only `include`

* switch to script

* Revert "switch to script"

This reverts commit 6135fc4.

* define matrix

* fmt

* I think I got it!

* Use alls-green

* add name to alls-green

Co-authored-by: Gregor Sturm <mail@gregor-sturm.de>

* Add comments to test.yaml

---------

Co-authored-by: Philipp A. <flying-sheep@web.de>

Author: grst

{{cookiecutter.project_name}}/.github/workflows/test.yaml

@@ -18,27 +18,48 @@ defaults:
     shell: bash -euo pipefail {0}
 
 jobs:
+  # Get the test environment from hatch as defined in pyproject.toml.
+  # This ensures that the pyproject.toml is the single point of truth for test definitions and the same tests are
+  # run locally and on continuous integration.
+  # Check [[tool.hatch.envs.hatch-test.matrix]] in pyproject.toml and https://hatch.pypa.io/latest/environment/ for
+  # more details.
+  get-environments:
+    runs-on: ubuntu-latest
+    outputs:
+      envs: ${{ steps.get-envs.outputs.envs }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          filter: blob:none
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Get test environments
+        id: get-envs
+        run: |
+          ENVS_JSON=$(uvx hatch env show --json | jq -c 'to_entries
+            | map(
+                select(.key | startswith("hatch-test"))
+                | {
+                    name: .key,
+                    label: (if (.key | contains("pre")) then .key + " (PRE-RELEASE DEPENDENCIES)" else .key end),
+                    python: .value.python
+                  }
+              )')
+          echo "envs=${ENVS_JSON}" | tee $GITHUB_OUTPUT
+
+  # Run tests through hatch. Spawns a separate runner for each environment defined in the hatch matrix obtained above.
   test:
-    runs-on: ${{ matrix.os }}
+    needs: get-environments
 
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-latest
-            python: "3.10"
-          - os: ubuntu-latest
-            python: "3.12"
-          - os: ubuntu-latest
-            python: "3.12"
-            pip-flags: "--pre"
-            name: PRE-RELEASE DEPENDENCIES
-
-    name: ${{ matrix.name }} Python ${{ matrix.python }}
-
-    env:
-      OS: ${{ matrix.os }}
-      PYTHON: ${{ matrix.python }}
+        os: [ubuntu-latest]
+        env: ${{ fromJSON(needs.get-environments.outputs.envs) }}
+
+    name: ${{ matrix.env.label }}
+    runs-on: ${{ matrix.os }}
 
     steps:
       - uses: actions/checkout@v4
@@ -48,14 +69,31 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
+          python-version: ${{ matrix.env.python }}
           cache-dependency-glob: pyproject.toml
+      - name: create hatch environment
+        run: uvx hatch env create ${{ matrix.env.name }}
       - name: run tests using hatch
         env:
           MPLBACKEND: agg
           PLATFORM: ${{ matrix.os }}
           DISPLAY: :42
-        run: uvx hatch test --cover --python ${{ matrix.python }}
+        run: uvx hatch run ${{ matrix.env.name }}:run-cov
       - name: generate coverage report
-        run: uvx hatch run hatch-test.py${{ matrix.python }}:coverage xml
+        run: uvx hatch run ${{ matrix.env.name }}:coverage xml
       - name: Upload coverage
         uses: codecov/codecov-action@v4
+
+  # Check that all tests defined above pass. This makes it easy to set a single "required" test in branch
+  # protection instead of having to update it frequently. See https://github.com/re-actors/alls-green#why.
+  check:
+    name: Tests pass in all hatch environments
+    if: always()
+    needs:
+      - get-environments
+      - test
+    runs-on: ubuntu-latest
+    steps:
+        - uses: re-actors/alls-green@release/v1
+          with:
+            jobs: ${{ toJSON(needs) }}

{{cookiecutter.project_name}}/pyproject.toml

@@ -72,6 +72,8 @@ scripts.build = "sphinx-build -M html docs docs/_build {args}"
 scripts.open = "python -m webbrowser -t docs/_build/html/index.html"
 scripts.clean = "git clean -fdX -- {args:docs}"
 
+[[tool.hatch.envs.hatch-test.matrix]]
+python = [ "3.10", "3.13" ]
 [tool.hatch.envs.hatch-test]
 features = [ "test" ]