Epic: MLX backend non-fitting parity #380
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # Python jobs (lint/test/build) skip when a change touches only prose or | |
| # metadata -- Markdown, the docs site, the JOSS paper (paper.md/paper.bib), and | |
| # citation/archive metadata (CITATION.cff, .zenodo.json, LICENSE) -- none of | |
| # which affect the package. Spell-checking (typos.yml) and the paper PDF build | |
| # (draft-pdf.yml) have their own triggers, so prose is still checked. | |
| # | |
| # push runs on both `main` (release branch) and `dev` (integration branch, the | |
| # default branch): post-merge pushes to dev are what catch drift that only | |
| # shows up outside a PR's exact base -- e.g. issue #241, where a ~1e-6 | |
| # trajectory shift moved a stopping iteration and only failed once the PR's | |
| # changes landed on dev. | |
| # | |
| # `concurrency` below keys on github.sha for push events but on github.ref for | |
| # pull_request events. For a PR, that keeps the original behavior: rapid pushes | |
| # to the same PR branch share one group, so an update cancels the older, | |
| # now-superseded run. For a push, every commit gets its OWN group instead -- | |
| # pushes never cancel each other -- because each commit landing on main/dev | |
| # needs its own CI result (that is the entire point of adding the push trigger | |
| # above); a same-ref grouping there would let a later push cancel an earlier | |
| # commit's still-running, still-meaningful CI. | |
| # | |
| # Interaction with auto-bump-dev.yml/auto-tag.yml (PR #290 review): both bump | |
| # the version and push straight back to dev/main with a PAT (deliberately, so | |
| # the push re-triggers workflows -- see their own headers). Per-SHA grouping | |
| # means that bump push can never land in the same concurrency group as the | |
| # merge commit's run, so it cannot cancel it -- the failure mode this section | |
| # used to describe (a rapid follow-up push cancelling the real commit's run | |
| # and leaving the trivial bump commit as the one that gets tested) cannot | |
| # happen at all now. `lint`/`typecheck` below still skip when the push's head | |
| # commit is a bot bump (same author-email + message-prefix guard | |
| # auto-bump-dev.yml/auto-tag.yml use on themselves to avoid looping), and every | |
| # other job `needs` one of those two so the skip cascades to the whole run -- | |
| # now purely to avoid spending compute on a run that would only re-verify | |
| # state the merge commit's own run already covered, not to prevent a | |
| # cancellation. | |
| # | |
| # The weekly full-suite run (slow tests + Fortran-parity, macOS) lives in its | |
| # own schedule-only workflow, .github/workflows/weekly-macos-slow.yml, so it | |
| # never gates a PR or a push here (issue #246). | |
| on: | |
| push: | |
| branches: [main, dev] | |
| paths-ignore: | |
| - "**.md" | |
| - "**.bib" | |
| - "docs/**" | |
| - "mkdocs.yml" | |
| - "CITATION.cff" | |
| - ".zenodo.json" | |
| - "LICENSE" | |
| pull_request: | |
| paths-ignore: | |
| - "**.md" | |
| - "**.bib" | |
| - "docs/**" | |
| - "mkdocs.yml" | |
| - "CITATION.cff" | |
| - ".zenodo.json" | |
| - "LICENSE" | |
| concurrency: | |
| # push: per-SHA group (each push to main/dev gets its own run, never | |
| # cancelled by a later push -- see the header comment above). | |
| # pull_request: per-ref group, unchanged (a PR's older, superseded runs are | |
| # still cancelled by a newer push to the same branch). | |
| group: ci-${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| name: Lint (ruff) | |
| runs-on: ubuntu-latest | |
| # Skip the workflow (see the header comment above) when the push's head | |
| # commit is auto-bump-dev.yml's or auto-tag.yml's own bump-and-push -- | |
| # matches the guard those workflows already use on themselves. | |
| if: | | |
| github.event.head_commit.author.email != 'pamica-bot@users.noreply.github.com' | |
| && !startsWith(github.event.head_commit.message, 'Bump version to') | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| - run: uv sync | |
| - name: ruff check | |
| run: uv run ruff check . | |
| - name: ruff format --check | |
| run: uv run ruff format --check . | |
| typecheck: | |
| name: Type check (ty) | |
| runs-on: ubuntu-latest | |
| # Same guard as lint above; every other job needs one of these two, so | |
| # GitHub cascades the skip to the whole run for a bot bump push. | |
| if: | | |
| github.event.head_commit.author.email != 'pamica-bot@users.noreply.github.com' | |
| && !startsWith(github.event.head_commit.message, 'Bump version to') | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| # ty is in the dev dependency group and version-locked via uv.lock, so | |
| # `uv sync` installs the same version used locally and by the pre-commit | |
| # hook. Optional-extra imports absent from the base env (mlx, mne) carry | |
| # scoped `# ty: ignore` comments, so this runs on the plain base env like | |
| # the other jobs (issue #119). | |
| - run: uv sync | |
| - name: ty check | |
| run: uv run ty check . | |
| test: | |
| name: Test (Python 3.12) | |
| needs: [lint, typecheck] | |
| runs-on: ubuntu-latest | |
| env: | |
| PYTORCH_ENABLE_MPS_FALLBACK: "1" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| - run: uv sync | |
| # Build the dependency-free native binary (epic #165) so the AMICANative | |
| # engine's end-to-end tests run for real on Linux (they skip without | |
| # PAMICA_NATIVE_BINARY). Static reference LAPACK/BLAS, same as the release. | |
| - name: Build native AMICA binary (for engine E2E tests) | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y gfortran liblapack-dev libblas-dev | |
| LAPACK_LIBS="-l:liblapack.a -l:libblas.a" EXTRA_LDFLAGS="-static-libquadmath" \ | |
| bash native/build.sh | |
| echo "PAMICA_NATIVE_BINARY=$PWD/native/amica15_shim" >> "$GITHUB_ENV" | |
| # Exclude the "slow" parity tests: they invoke the macOS-only Fortran | |
| # reference binary (pamica/sample_data/amica15mac), which cannot run on | |
| # the Linux runner. MPS-specific tests self-skip when MPS is absent. | |
| # -n auto parallelizes the (independent) torch fits across runner cores; | |
| # pytest-cov combines per-worker branch coverage automatically. Coverage | |
| # config + the fail-under gate live in pyproject.toml / the flag below. | |
| - name: pytest (excluding slow / Fortran-binary parity tests) | |
| run: uv run pytest -m "not slow" -n auto --cov-fail-under=80 | |
| - name: Upload coverage HTML report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-html | |
| path: htmlcov/ | |
| retention-days: 14 | |
| # Codecov is informational (badge + PR coverage-diff comments); the hard | |
| # gate is --cov-fail-under in the pytest step above, so a flaky upload | |
| # (fail_ci_if_error: false) never breaks the build. | |
| - name: Upload coverage to Codecov | |
| if: always() | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: coverage.xml | |
| fail_ci_if_error: false | |
| # Apple Silicon coverage (issue #246). The Linux jobs above cannot exercise | |
| # three things the project depends on: | |
| # * pamica/tests/mlx_tests/ -- MLX is Apple-GPU only and is the recommended | |
| # Apple backend (AGENTS.md), so it had never run in CI at all. | |
| # * the Accelerate BLAS numerical path. Summation order differs from | |
| # OpenBLAS enough to move a stopping iteration, which is how a broken | |
| # assertion in test_ng_convergence.py stayed green on Linux through #241 | |
| # and failed immediately on macOS. | |
| # * pamica/tests/mne_tests/ (the AMICAICA/MNE wrapper, issue #139) -- | |
| # previously exercised only by test-mne below, which is Linux/OpenBLAS | |
| # only. Installing --extra mne here runs the same suite against | |
| # Accelerate; per #241's precedent a real Accelerate-vs-OpenBLAS | |
| # divergence there is a genuine finding, not CI flakiness -- report it, | |
| # do not loosen the assertion to mask it. | |
| # macos-26 is the current arm64 image (macos-14/15 are also arm64; 14 is | |
| # deprecated). Standard runners are free and unmetered for public repos -- | |
| # do not switch to a -large/-xlarge variant, which is billed even here. | |
| test-macos: | |
| name: Test (macOS 26, Apple Silicon) | |
| needs: [lint, typecheck] | |
| runs-on: macos-26 | |
| env: | |
| # The only job on hardware where torch auto-selects MPS (torch_impl/ | |
| # utils.py picks it whenever no device is pinned and MPS is available), | |
| # so it is the only job where the fallback actually does anything. | |
| PYTORCH_ENABLE_MPS_FALLBACK: "1" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| # --extra mlx is the whole point of this job: without it mlx_tests/ | |
| # self-skips and the run adds nothing the Linux job did not already do. | |
| # --extra mne (issue #246) makes pamica/tests/mne_tests/ (self-skipped | |
| # via importorskip("mne") otherwise) run against Accelerate instead of | |
| # only ever against Linux/OpenBLAS in test-mne below. | |
| - run: uv sync --extra mlx --extra mne | |
| - name: Assert MLX sees an Apple GPU | |
| # mlx_tests/ is gated on BOTH an mlx import and | |
| # mx.default_device().type == gpu (test_mlx_backend.py:28-33). A hosted | |
| # runner that imported mlx but fell back to CPU would skip all 11 tests | |
| # and still report success -- green while testing nothing, the exact | |
| # failure this job exists to prevent. Assert the device, not just the | |
| # import, so that regression fails here instead of passing silently. | |
| run: | | |
| uv run python -c " | |
| import mlx.core as mx | |
| dev = mx.default_device() | |
| print('mlx', mx.__version__, 'device', dev) | |
| assert dev.type == mx.DeviceType.gpu, f'no Apple GPU: {dev}' | |
| " | |
| - name: Build native AMICA binary (for engine E2E tests) | |
| # Same recipe as the macos-arm64 target in release-binaries.yml: | |
| # gfortran from brew's gcc, LAPACK/BLAS from Accelerate, and the static | |
| # libquadmath the release build uses. | |
| run: | | |
| brew install gcc | |
| LAPACK_LIBS="-framework Accelerate" EXTRA_LDFLAGS="-static-libquadmath" \ | |
| bash native/build.sh | |
| echo "PAMICA_NATIVE_BINARY=$PWD/native/amica15_shim" >> "$GITHUB_ENV" | |
| # "not slow" matches the Linux job, but NOT for the Linux job's reason. | |
| # There it is because the slow tests need the macOS-only reference binary; | |
| # here they are excluded purely for wall-clock (2000-iteration fits) -- | |
| # this job still runs on every push/PR and stays fast. The tests that do | |
| # shell out (e.g. test_ng_backend.py:1096,:1179) pass binary_path=None, | |
| # which resolves via PAMICA_NATIVE_BINARY (native/resolver.py:112) -- | |
| # set above -- so they use the arm64 binary this job just built, never | |
| # Rosetta; they just aren't run here because they are @pytest.mark.slow. | |
| # The weekly schedule-only job (weekly-macos-slow.yml, issue #246) runs | |
| # the full suite without this filter, giving CI its first Fortran | |
| # parity + slow-test coverage, without slowing every PR down. | |
| # pamica/tests/mne_tests/ is picked up here too (pytest.ini's testpaths | |
| # covers all of pamica/tests) now that --extra mne is installed above. | |
| # --no-cov: the Linux job owns the coverage gate and the Codecov upload; | |
| # a second partial run would report a different total for the same commit. | |
| - name: pytest (excluding slow tests) | |
| run: uv run pytest -m "not slow" -n auto --no-cov | |
| test-mne: | |
| name: Test MNE wrapper (optional [mne] extra) | |
| needs: [lint, typecheck] | |
| runs-on: ubuntu-latest | |
| env: | |
| PYTORCH_ENABLE_MPS_FALLBACK: "1" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| # mne IS pip-installable on Linux (unlike the Apple-only mlx extra), so the | |
| # AMICAICA wrapper (issue #139) is exercised for real in CI here. Its tests | |
| # live in pamica/tests/mne_tests and self-skip in the base `test` job | |
| # (importorskip on mne); this job installs the [mne] extra so they run on | |
| # the real sample EEG. pamica/mne_compat is omitted from coverage in | |
| # pyproject (the base env has no mne), so no fail-under gate applies here. | |
| - run: uv sync --extra mne | |
| - name: pytest (mne wrapper, real sample EEG) | |
| run: uv run pytest pamica/tests/mne_tests | |
| build: | |
| name: Build + import (Python ${{ matrix.python-version }}) | |
| needs: [lint, typecheck] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.12", "3.13"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| enable-cache: true | |
| - name: Build sdist + wheel | |
| run: uv build | |
| # Verify requires-python>=3.12 compatibility: install the built wheel and | |
| # its declared dependencies into a clean environment and import it. | |
| - name: Install wheel into a clean env and import | |
| run: | | |
| uv venv --python ${{ matrix.python-version }} .import-venv | |
| uv pip install --python .import-venv dist/*.whl | |
| .import-venv/bin/python -c "import pamica; print('pamica', pamica.__version__)" |