docs(bench): refresh benchmarks against emlx 0.4.1 #465
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| # `feat/expr-compiler` is the long-lived integration branch for the | |
| # Nx.Defn.Expr -> MLX compiler effort; milestone branches merge into | |
| # it via PR and it merges to main once. Run CI on pushes to it (post | |
| # -merge validation) and on PRs into it (every milestone). Remove | |
| # both entries when that branch merges to main and is deleted. | |
| branches: [main, feat/expr-compiler] | |
| tags: ['[0-9]+.[0-9]+.[0-9]+*'] | |
| pull_request: | |
| branches: [main, feat/expr-compiler] | |
| workflow_dispatch: | |
| inputs: | |
| run_full_tests: | |
| description: 'Run full model tests (whisper_full, vit_full, distilbert_full, training_full, fast_kernels_full)' | |
| type: boolean | |
| default: true | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| precommit: | |
| name: precommit (${{ matrix.os }}, variant=${{ matrix.variant }}) | |
| if: github.event_name != 'push' || github.ref_type == 'branch' | |
| # AOT prebuilt is built on macos-14 and is the older-macOS-compatible | |
| # path. JIT prebuilt is built on macos-26 and isn't portable back to | |
| # macos-14 (the macOS 26 libSystem grew `__fmaxf16` and other half- | |
| # float intrinsics that MLX's Metal stack ends up referencing). Each | |
| # lane therefore runs on the runner whose SDK the prebuilt targets. | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: macos-14 | |
| variant: aot | |
| min: "14.0" | |
| - os: macos-26 | |
| variant: jit | |
| min: "26.2" | |
| env: | |
| MIX_ENV: test | |
| EMILY_MLX_VARIANT: ${{ matrix.variant }} | |
| # Pin the cache to the path the actions/cache step below | |
| # populates. Without it, mix.exs's macOS default of | |
| # DARWIN_USER_CACHE_DIR (`/private/var/folders/...`) misses | |
| # the restored cache and rebuilds MLX from scratch every run. | |
| # mix.exs runs `Path.expand` on this, so `~` resolves correctly. | |
| EMILY_CACHE: ~/Library/Caches/emily | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Beam | |
| uses: erlef/setup-beam@v1 | |
| with: | |
| version-file: .tool-versions | |
| version-type: strict | |
| - name: Cache deps | |
| uses: actions/cache@v5 | |
| with: | |
| path: deps | |
| key: deps-${{ runner.os }}-${{ hashFiles('mix.lock') }} | |
| restore-keys: deps-${{ runner.os }}- | |
| - name: Cache build | |
| uses: actions/cache@v5 | |
| with: | |
| path: _build | |
| key: build-${{ runner.os }}-${{ matrix.variant }}-${{ hashFiles('mix.lock', 'c_src/**', 'Makefile', '.tool-versions', 'mix.exs', 'config/config.exs') }} | |
| restore-keys: build-${{ runner.os }}-${{ matrix.variant }}- | |
| # MLX prebuilt tarball (~40 MB AOT / ~5 MB JIT) + extracted install. | |
| # Keyed by the NIF sources and the pinned MLX version in mix.exs; | |
| # variants are cached separately because the install dir differs. | |
| - name: Cache MLX and NIF objects | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/Library/Caches/emily | |
| key: mlx-${{ runner.os }}-${{ matrix.variant }}-${{ hashFiles('c_src/**', 'Makefile', 'mix.exs', 'scripts/build-mlx.sh') }} | |
| restore-keys: mlx-${{ runner.os }}-${{ matrix.variant }}- | |
| # The Bumblebee cache holds tiny-random HuggingFace fixtures | |
| # downloaded by the conformance suite (~3 MB across 7 repos). | |
| # Cached by conformance test content so new fixtures invalidate; | |
| # restore-keys lets unrelated changes reuse the existing cache. | |
| - name: Cache Bumblebee fixtures | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/Library/Caches/bumblebee | |
| key: bumblebee-${{ runner.os }}-${{ hashFiles('test/emily/conformance/**') }} | |
| restore-keys: bumblebee-${{ runner.os }}- | |
| - run: mix deps.get | |
| - run: mix precommit | |
| # Assert the built NIF declares the pinned macOS floor for this variant | |
| # and imports no above-floor libSystem symbols (see the script). Catches | |
| # a variant that silently requires a newer macOS than it targets. | |
| - name: Verify NIF macOS floor | |
| run: bash scripts/verify-nif-floor.sh _build/test/lib/emily/priv/libemily.so "${{ matrix.min }}" | |
| # Conformance tests are excluded from the default suite because | |
| # they require network access on a cold cache (see | |
| # test/test_helper.exs). In CI we always want them green — a | |
| # DistilBERT forward pass is the canonical integration signal | |
| # that no Nx op on the transformer critical path has regressed. | |
| - run: mix test --only conformance | |
| # Manual-only job: exercises the heavy `*_full` conformance and | |
| # training variants excluded from the default suite (see | |
| # test/test_helper.exs). These download full-size checkpoints | |
| # (hundreds of MB) and take multi-minute wall time, so they run | |
| # on demand via the Actions "Run workflow" button rather than on | |
| # every push / PR. | |
| full-tests: | |
| name: full model tests (${{ matrix.os }}, variant=${{ matrix.variant }}) | |
| if: github.event_name == 'workflow_dispatch' && inputs.run_full_tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: macos-14 | |
| variant: aot | |
| min: "14.0" | |
| - os: macos-26 | |
| variant: jit | |
| min: "26.2" | |
| env: | |
| MIX_ENV: test | |
| EMILY_MLX_VARIANT: ${{ matrix.variant }} | |
| # Pin the cache to the path the actions/cache step below | |
| # populates. Without it, mix.exs's macOS default of | |
| # DARWIN_USER_CACHE_DIR (`/private/var/folders/...`) misses | |
| # the restored cache and rebuilds MLX from scratch every run. | |
| # mix.exs runs `Path.expand` on this, so `~` resolves correctly. | |
| EMILY_CACHE: ~/Library/Caches/emily | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Beam | |
| uses: erlef/setup-beam@v1 | |
| with: | |
| version-file: .tool-versions | |
| version-type: strict | |
| - name: Cache deps | |
| uses: actions/cache@v5 | |
| with: | |
| path: deps | |
| key: deps-${{ runner.os }}-${{ hashFiles('mix.lock') }} | |
| restore-keys: deps-${{ runner.os }}- | |
| - name: Cache build | |
| uses: actions/cache@v5 | |
| with: | |
| path: _build | |
| key: build-${{ runner.os }}-${{ matrix.variant }}-${{ hashFiles('mix.lock', 'c_src/**', 'Makefile', '.tool-versions', 'mix.exs', 'config/config.exs') }} | |
| restore-keys: build-${{ runner.os }}-${{ matrix.variant }}- | |
| - name: Cache MLX and NIF objects | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/Library/Caches/emily | |
| key: mlx-${{ runner.os }}-${{ matrix.variant }}-${{ hashFiles('c_src/**', 'Makefile', 'mix.exs', 'scripts/build-mlx.sh') }} | |
| restore-keys: mlx-${{ runner.os }}-${{ matrix.variant }}- | |
| # Full checkpoints land here (Bumblebee + scidata MNIST). | |
| - name: Cache Bumblebee fixtures | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/Library/Caches/bumblebee | |
| key: bumblebee-full-${{ runner.os }}-${{ hashFiles('test/emily/conformance/**', 'test/emily/bumblebee/**') }} | |
| restore-keys: | | |
| bumblebee-full-${{ runner.os }}- | |
| bumblebee-${{ runner.os }}- | |
| - run: mix deps.get | |
| - run: mix compile | |
| - name: whisper_full | |
| run: mix test --only whisper_full | |
| - name: vit_full | |
| run: mix test --only vit_full | |
| - name: distilbert_full | |
| run: mix test --only distilbert_full | |
| - name: training_full | |
| run: mix test --only training_full | |
| - name: fast_kernels_full | |
| run: mix test --only fast_kernels_full | |
| # C++ static analysis of the NIF sources with cppcheck. Kept separate | |
| # from the precommit lane on purpose: the `cppcheck` Makefile target | |
| # needs neither a built libmlx nor the BEAM toolchain (it analyses our | |
| # own c_src/ tree and suppresses the third-party headers it can't see), | |
| # so this runs on a bare checkout and finishes in seconds. Runs on | |
| # macOS to match the platform the NIF actually ships on, and invokes | |
| # the same `make cppcheck` a developer runs locally. | |
| cppcheck: | |
| name: cppcheck (NIF static analysis) | |
| if: github.event_name != 'push' || github.ref_type == 'branch' | |
| runs-on: macos-14 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Install cppcheck | |
| run: brew install cppcheck | |
| - name: Run cppcheck | |
| run: make cppcheck | |
| # C++ static analysis of the NIF sources with clang-tidy (which also runs | |
| # the clang static analyzer via its clang-analyzer-* checks). Unlike | |
| # cppcheck, clang-tidy compiles each TU, so it needs the MLX/Fine/ERTS | |
| # headers — hence the Beam setup and the MLX cache. It reuses the MLX | |
| # install the precommit lane builds: the restore-keys pick up main's | |
| # cache even when this branch's Makefile hash differs, since only the | |
| # headers are needed and the MLX version is unchanged. clang-tidy ships | |
| # with LLVM (brew), not stock Xcode. Runs on macOS to match the platform | |
| # the NIF ships on, via the same `mix clang.tidy` a developer runs. | |
| clang-tidy: | |
| name: clang-tidy (NIF static analysis) | |
| if: github.event_name != 'push' || github.ref_type == 'branch' | |
| runs-on: macos-14 | |
| env: | |
| MIX_ENV: test | |
| EMILY_MLX_VARIANT: aot | |
| EMILY_CACHE: ~/Library/Caches/emily | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Beam | |
| uses: erlef/setup-beam@v1 | |
| with: | |
| version-file: .tool-versions | |
| version-type: strict | |
| - name: Install LLVM (clang-tidy) | |
| run: brew install llvm | |
| - name: Cache deps | |
| uses: actions/cache@v5 | |
| with: | |
| path: deps | |
| key: deps-${{ runner.os }}-${{ hashFiles('mix.lock') }} | |
| restore-keys: deps-${{ runner.os }}- | |
| - name: Cache MLX and NIF objects | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/Library/Caches/emily | |
| key: mlx-${{ runner.os }}-aot-${{ hashFiles('c_src/**', 'Makefile', 'mix.exs', 'scripts/build-mlx.sh') }} | |
| restore-keys: mlx-${{ runner.os }}-aot- | |
| - run: mix deps.get | |
| - name: Run clang-tidy | |
| run: | | |
| export CLANG_TIDY="$(brew --prefix llvm)/bin/clang-tidy" | |
| export SDKROOT="$(xcrun --show-sdk-path)" | |
| mix clang.tidy | |
| # ASan CI deferred: requires OTP built with --enable-sanitizers=address | |
| # (macOS SIP blocks DYLD_INSERT_LIBRARIES, and late-loaded libasan | |
| # fails). See Makefile and RELEASE.md for details. |