Merge pull request #55 from quadbio/fix/codecov

Update test workflow

Author: Marius1311

.github/workflows/test.yaml

@@ -18,23 +18,48 @@ defaults:
     shell: bash -euo pipefail {0}
 
 jobs:
+  # Get the test environment from hatch as defined in pyproject.toml.
+  # This ensures that the pyproject.toml is the single point of truth for test definitions and the same tests are
+  # run locally and on continuous integration.
+  # Check [[tool.hatch.envs.hatch-test.matrix]] in pyproject.toml and https://hatch.pypa.io/latest/environment/ for
+  # more details.
+  get-environments:
+    runs-on: ubuntu-latest
+    outputs:
+      envs: ${{ steps.get-envs.outputs.envs }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          filter: blob:none
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Get test environments
+        id: get-envs
+        run: |
+          ENVS_JSON=$(uvx hatch env show --json | jq -c 'to_entries
+            | map(
+                select(.key | startswith("hatch-test"))
+                | {
+                    name: .key,
+                    label: (if (.key | contains("pre")) then .key + " (PRE-RELEASE DEPENDENCIES)" else .key end),
+                    python: .value.python
+                  }
+              )')
+          echo "envs=${ENVS_JSON}" | tee $GITHUB_OUTPUT
+
+  # Run tests through hatch. Spawns a separate runner for each environment defined in the hatch matrix obtained above.
   test:
-    runs-on: ${{ matrix.os }}
+    needs: get-environments
 
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-latest
-            python: "3.10"
-          - os: ubuntu-latest
-            python: "3.12"
-          - os: ubuntu-latest
-            python: "3.12"
-            pip-flags: "--pre"
-            name: PRE-RELEASE DEPENDENCIES
+        os: [ubuntu-latest]
+        env: ${{ fromJSON(needs.get-environments.outputs.envs) }}
 
-    name: ${{ matrix.name }} Python ${{ matrix.python }}
+    name: ${{ matrix.env.label }}
+    runs-on: ${{ matrix.os }}
 
     env:
       OS: ${{ matrix.os }}
@@ -51,25 +76,33 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
+          python-version: ${{ matrix.env.python }}
           cache-dependency-glob: pyproject.toml
-      - name: Install coverage
-        run: pip install coverage
+      - name: create hatch environment
+        run: uvx hatch env create ${{ matrix.env.name }}
       - name: run tests using hatch
         env:
           MPLBACKEND: agg
           PLATFORM: ${{ matrix.os }}
           DISPLAY: :42
-        run: |
-          if [ "${{ matrix.python }}" == "3.12" ] && [ -z "${{ matrix.pip-flags }}" ]; then
-            uvx hatch test --cover --python ${{ matrix.python }}
-          else
-            uvx hatch test --cover --python ${{ matrix.python }} -m "not openai"
-          fi
-      - name: Report coverage
-        run: |
-          coverage report
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v5
+        run: uvx hatch run ${{ matrix.env.name }}:run-cov
+      - name: generate coverage report
+        run: uvx hatch run ${{ matrix.env.name }}:coverage xml
+      - name: Upload coverage
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          slug: quadbio/cell-annotator
+
+  # Check that all tests defined above pass. This makes it easy to set a single "required" test in branch
+  # protection instead of having to update it frequently. See https://github.com/re-actors/alls-green#why.
+  check:
+    name: Tests pass in all hatch environments
+    if: always()
+    needs:
+      - get-environments
+      - test
+    runs-on: ubuntu-latest
+    steps:
+        - uses: re-actors/alls-green@release/v1
+          with:
+            jobs: ${{ toJSON(needs) }}

pyproject.toml

@@ -108,9 +108,26 @@ scripts.build = "sphinx-build -M html docs docs/_build {args}"
 scripts.open = "python -m webbrowser -t docs/_build/html/index.html"
 scripts.clean = "git clean -fdX -- {args:docs}"
 
+# Test the lowest and highest supported Python versions with normal deps
+[[tool.hatch.envs.hatch-test.matrix]]
+deps = [ "stable" ]
+python = [ "3.10", "3.13" ]
+
+# Test the newest supported Python version also with pre-release deps
+[[tool.hatch.envs.hatch-test.matrix]]
+deps = [ "pre" ]
+python = [ "3.13" ]
+
 [tool.hatch.envs.hatch-test]
 features = [ "test" ]
 
+[tool.hatch.envs.hatch-test.overrides]
+# If the matrix variable `deps` is set to "pre",
+# set the environment variable `UV_PRERELEASE` to "allow".
+matrix.deps.env-vars = [
+  { key = "UV_PRERELEASE", value = "allow", if = [ "pre" ] },
+]
+
 [tool.ruff]
 line-length = 120
 src = [ "src" ]

tests/model/test_base_annotator.py

@@ -1,6 +1,7 @@
 from unittest.mock import patch
 
 import pytest
+from flaky import flaky
 
 from cell_annotator._response_formats import OutputForTesting
 from cell_annotator.model.base_annotator import BaseAnnotator
@@ -24,6 +25,7 @@ def test_query_llm_mock(self, mock_query_llm, base_annotator):
             response_format=OutputForTesting,
         )
 
+    @flaky
     @pytest.mark.real_llm_query()
     def test_query_llm_real(self, base_annotator):
         """Test actual query_llm call across all available providers."""

tests/model/test_cell_annotator.py

@@ -4,6 +4,7 @@
 
 
 class TestCellAnnotator:
+    @flaky
     @pytest.mark.real_llm_query()
     def test_get_expected_cell_type_markers(self, cell_annotator_single):
         """Test getting expected cell type markers with single sample data."""
@@ -36,6 +37,7 @@ def test_get_expected_cell_type_markers(self, cell_annotator_single):
         assert neuron_markers_found
         assert fibroblast_markers_found
 
+    @flaky
     @pytest.mark.real_llm_query()
     def test_annotate_clusters_single(self, cell_annotator_single):
         """Test annotating clusters with single sample data."""

tests/model/test_obs_beautifier.py

@@ -1,11 +1,13 @@
 import numpy as np
 import pandas as pd
 import pytest
+from flaky import flaky
 
 from cell_annotator.model.obs_beautifier import ObsBeautifier
 
 
 class TestObsBeautifier:
+    @flaky
     @pytest.mark.real_llm_query()
     def test_reorder_and_color_clusters(self, cell_annotator_single):
         """Test reordering and coloring clusters."""
@@ -30,6 +32,7 @@ def test_reorder_and_color_clusters(self, cell_annotator_single):
             assert f"{key}_colors" in cell_annotator.adata.uns
             assert len(cell_annotator.adata.uns[f"{key}_colors"]) == cell_annotator.adata.obs[key].nunique()
 
+    @flaky
     @pytest.mark.real_llm_query()
     def test_reorder_only_preserves_colors(self, cell_annotator_single):
         """Test that reordering clusters without assigning new colors preserves the original colors."""

tests/model/test_providers.py

@@ -3,6 +3,7 @@
 import os
 
 import pytest
+from flaky import flaky
 
 from cell_annotator._response_formats import BaseOutput
 from cell_annotator.model._providers import AnthropicProvider, GeminiProvider, OpenAIProvider
@@ -65,6 +66,7 @@ def test_provider_repr(self):
 class TestOpenAIProvider:
     """Isolated tests for OpenAI provider."""
 
+    @flaky
     @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OPENAI_API_KEY not available")
     @pytest.mark.real_llm_query()
     def test_openai_list_models_real(self):
@@ -78,6 +80,7 @@ def test_openai_list_models_real(self):
         model_names = [model.lower() for model in models]
         assert any("gpt" in model for model in model_names)
 
+    @flaky
     @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OPENAI_API_KEY not available")
     @pytest.mark.real_llm_query()
     def test_openai_query_real(self):
@@ -118,6 +121,7 @@ def test_openai_query_error_handling(self):
 class TestGeminiProvider:
     """Isolated tests for Gemini provider."""
 
+    @flaky
     @pytest.mark.skipif(not os.getenv("GEMINI_API_KEY"), reason="GEMINI_API_KEY not available")
     @pytest.mark.real_llm_query()
     def test_gemini_list_models_real(self):
@@ -131,6 +135,7 @@ def test_gemini_list_models_real(self):
         model_names = [model.lower() for model in models]
         assert any("gemini" in model for model in model_names)
 
+    @flaky
     @pytest.mark.skipif(not os.getenv("GEMINI_API_KEY"), reason="GEMINI_API_KEY not available")
     @pytest.mark.real_llm_query()
     def test_gemini_query_real(self):
@@ -163,6 +168,7 @@ def test_gemini_initialization(self):
 class TestAnthropicProvider:
     """Isolated tests for Anthropic provider."""
 
+    @flaky
     @pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="ANTHROPIC_API_KEY not available")
     @pytest.mark.real_llm_query()
     def test_anthropic_list_models_real(self):
@@ -176,6 +182,7 @@ def test_anthropic_list_models_real(self):
         model_names = [model.lower() for model in models]
         assert any("claude" in model for model in model_names)
 
+    @flaky
     @pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="ANTHROPIC_API_KEY not available")
     @pytest.mark.real_llm_query()
     def test_anthropic_query_real(self):
@@ -221,6 +228,7 @@ def test_provider_factory_pattern(self):
             assert provider is not None
             assert provider.__class__.__name__.lower().startswith(name)
 
+    @flaky
     @pytest.mark.skipif(
         not any(os.getenv(key) for key in ["OPENAI_API_KEY", "GEMINI_API_KEY", "ANTHROPIC_API_KEY"]),
         reason="No API keys available for testing",

tests/model/test_sample_annotator.py

@@ -2,16 +2,10 @@
 
 import pandas as pd
 import pytest
-from tests.utils import (
-    expected_marker_genes,
-    fibroblast_cell_types,
-    neuronal_cell_types,
-)
+from flaky import flaky
+from tests.utils import expected_marker_genes, fibroblast_cell_types, neuronal_cell_types
 
-from cell_annotator._response_formats import (
-    CellTypeMappingOutput,
-    PredictedCellTypeOutput,
-)
+from cell_annotator._response_formats import CellTypeMappingOutput, PredictedCellTypeOutput
 
 
 class TestSampleAnnotator:
@@ -59,6 +53,7 @@ def test_get_cluster_markers(self, sample_annotator):
         for _cluster, genes in sample_annotator.marker_genes.items():
             assert len(genes) > 0
 
+    @flaky
     @pytest.mark.real_llm_query()
     def test_annotate_clusters_actual(self, sample_annotator):
         sample_annotator.get_cluster_markers(min_auc=0.6)