[anneal][v2] Implement out-of-tree dependency chasing for expand command #1271
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2026 The Fuchsia Authors | |
| # | |
| # Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 | |
| # <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT | |
| # license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. | |
| # This file may not be copied, modified, or distributed except according to | |
| # those terms. | |
| name: Anneal Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| merge_group: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUSTFLAGS: -Dwarnings | |
| RUSTDOCFLAGS: -Dwarnings | |
| CARGO_ZEROCOPY_AUTO_INSTALL_TOOLCHAIN: 1 | |
| jobs: | |
| static_checks: | |
| name: Anneal Static Checks | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - name: Install stable Rust | |
| uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[superfluous-actions] | |
| with: | |
| toolchain: stable | |
| - name: Test Anneal support scripts | |
| run: | | |
| set -euo pipefail | |
| export PYTHONDONTWRITEBYTECODE=1 | |
| python3 -m py_compile \ | |
| anneal/tools/check-release-pr-files.py \ | |
| anneal/tools/test_exocrate_metadata_helpers.py \ | |
| anneal/tools/test_release_pr_files.py \ | |
| anneal/tools/collect-release-archive-metadata.py \ | |
| anneal/tools/update-exocrate-metadata.py \ | |
| anneal/v2/tests/test_prune_lake_cache.py \ | |
| anneal/v2/prune-lake-cache.py \ | |
| anneal/v2/rewrite-lake-vendor.py | |
| python3 -m unittest discover -s anneal/tools -p 'test_*.py' | |
| python3 -m unittest discover -s anneal/v2/tests -p 'test_*.py' | |
| bash anneal/tools/check-release-flow-dry-run.sh | |
| - name: Install Nix | |
| uses: DeterminateSystems/determinate-nix-action@4eea0b33e3d1f02ecfe37cf16e7204c424009606 # v3.21.0 | |
| - name: Check V2 flake evaluation | |
| run: bash anneal/v2/check-flake-eval.sh | |
| anneal_tests: | |
| name: Anneal Tests | |
| runs-on: ubuntu-latest | |
| needs: v2_nix_cache | |
| permissions: | |
| actions: read # Required to download the toolchain artifact. | |
| contents: write # Required to push benchmark data to the storage branch | |
| env: | |
| ANNEAL_TOOLCHAIN_DIR: ${{ github.workspace }}/anneal/target/anneal-toolchain | |
| __ZEROCOPY_LOCAL_DEV: 1 | |
| RUSTFLAGS: "" | |
| RUSTDOCFLAGS: "" | |
| RUST_TEST_THREADS: "1" | |
| steps: | |
| - name: Record job start time | |
| id: job_start_time | |
| run: echo "unix=$(date +%s)" >> "$GITHUB_OUTPUT" | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - name: Install stable Rust | |
| uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[superfluous-actions] | |
| with: | |
| toolchain: stable | |
| - name: Download Anneal toolchain archive | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: anneal-exocrate.tar.zst | |
| path: anneal/v2/target | |
| # Ensure `llms-full.txt` file is up-to-date. | |
| - name: Check doc generation | |
| run: cargo run -p doc_gen -- --check | |
| working-directory: anneal | |
| - name: Install Anneal toolchain archive | |
| run: cargo run setup --local-archive v2/target/anneal-exocrate.tar.zst | |
| working-directory: anneal | |
| # Run unit tests separately, as they're much less likely to have bugs | |
| # during local development, and this makes the GitHub Actions output | |
| # easier to skim (in particular, it's clear at a glance whether a failure | |
| # is due to unit or integration tests). | |
| - name: Run unit tests | |
| run: cargo test --verbose --bin cargo-anneal | |
| working-directory: anneal | |
| # We duplicate running unit tests since they're very cheap compared to | |
| # integration tests, and this way it's easier to be sure that we run all | |
| # tests instead of specifically trying to carve out unit tests and risk | |
| # missing test categories. | |
| - name: Run all tests | |
| run: | | |
| start=$(date +%s) | |
| cargo test --verbose | |
| end=$(date +%s) | |
| duration=$((end - start)) | |
| echo "Test Time: $duration seconds" | |
| echo "[{\"name\": \"Test Time\", \"unit\": \"seconds\", \"value\": $duration}]" > test_time.json | |
| working-directory: anneal | |
| - name: Combine benchmarks | |
| env: | |
| JOB_START_TIME_UNIX: ${{ steps.job_start_time.outputs.unix }} | |
| run: | | |
| total_duration=$(( $(date +%s) - JOB_START_TIME_UNIX )) | |
| echo "Total CI Duration (All Steps): $total_duration seconds" | |
| echo "[{\"name\": \"Total CI Duration (All Steps)\", \"unit\": \"seconds\", \"value\": $total_duration}]" > total_time.json | |
| jq -n \ | |
| --slurpfile test anneal/test_time.json \ | |
| --slurpfile total total_time.json \ | |
| '[ | |
| $test[0][0], | |
| $total[0][0] | |
| ]' > output.json | |
| - name: Store CI duration benchmarks | |
| # Only trusted main-branch pushes to the canonical repository can | |
| # update the benchmark-data branch. Pull requests from forks receive a | |
| # read-only GITHUB_TOKEN, and forks running this workflow should not try | |
| # to publish benchmark history for google/zerocopy. | |
| if: github.event_name == 'push' && github.repository == 'google/zerocopy' && github.ref == 'refs/heads/main' | |
| uses: benchmark-action/github-action-benchmark@52576c92bccf6ac60c8223ec7eb2565637cae9ba # v1.22.1 | |
| with: | |
| name: CI Durations | |
| tool: 'customSmallerIsBetter' | |
| output-file-path: output.json | |
| gh-pages-branch: benchmark-data | |
| auto-push: true | |
| save-data-file: true | |
| benchmark-data-dir-path: dashboard | |
| fail-on-alert: false | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| verify_examples: | |
| name: Verify example (${{ matrix.example }}) | |
| runs-on: ubuntu-latest | |
| needs: v2_nix_cache | |
| permissions: | |
| actions: read # Required to download the toolchain artifact. | |
| contents: read | |
| env: | |
| ANNEAL_TOOLCHAIN_DIR: ${{ github.workspace }}/anneal/target/anneal-toolchain | |
| __ZEROCOPY_LOCAL_DEV: 1 | |
| RUSTFLAGS: "" | |
| RUSTDOCFLAGS: "" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| example: | |
| - abs | |
| - anatomy | |
| - checked_add | |
| - const_generics | |
| - design_doc | |
| - linked_list | |
| - namespaces | |
| - never_type | |
| - ptr_concat | |
| - size_of_align_of | |
| - swap | |
| - unchecked_get | |
| - update_max | |
| steps: | |
| - name: Free up disk space | |
| run: | | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/share/boost | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - name: Install stable Rust | |
| uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[superfluous-actions] | |
| with: | |
| toolchain: stable | |
| - name: Download Anneal toolchain archive | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: anneal-exocrate.tar.zst | |
| path: anneal/v2/target | |
| - name: Install Anneal toolchain archive | |
| run: cargo run setup --local-archive v2/target/anneal-exocrate.tar.zst | |
| working-directory: anneal | |
| - name: Verify example | |
| env: | |
| EXAMPLE: ${{ matrix.example }} | |
| run: | | |
| KNOWN_FAILING=("design_doc" "never_type" "ptr_concat") | |
| example="$EXAMPLE" | |
| expect_failure=0 | |
| for kf in "${KNOWN_FAILING[@]}"; do | |
| if [ "$kf" = "$example" ]; then | |
| expect_failure=1 | |
| break | |
| fi | |
| done | |
| echo "Verifying $example (expect failure: $expect_failure)" | |
| if cargo run verify --unsound-allow-is-valid --example "$example"; then | |
| if [ "$expect_failure" -eq 1 ]; then | |
| echo "::error::Example $example succeeded but was expected to fail." | |
| exit 1 | |
| else | |
| echo "Example $example succeeded." | |
| fi | |
| else | |
| if [ "$expect_failure" -eq 1 ]; then | |
| echo "Example $example failed as expected." | |
| else | |
| echo "::error::Example $example failed." | |
| exit 1 | |
| fi | |
| fi | |
| working-directory: anneal | |
| # Build the Nix-produced toolchain archive once and fan out the exact archive | |
| # as a workflow artifact. This avoids forcing every downstream matrix runner | |
| # to realize the same Nix closure through Magic Nix Cache, which can run into | |
| # GitHub Actions cache throttling under parallel fan-out. | |
| v2_nix_cache: | |
| name: Build Anneal Toolchain Archive | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| # Restore a local Nix binary cache populated only by trusted `main` | |
| # pushes. Pull requests and merge-queue runs may read the default-branch | |
| # cache, but they never save their own entries there; that keeps sibling | |
| # PRs from publishing toolchain archives for each other. This cache is | |
| # only for cross-run reuse in the builder job. The workflow artifact below | |
| # remains the cross-job fan-out mechanism for the current run. | |
| # | |
| # Keep the key inputs in sync with the local files that influence | |
| # `.#omnibus-archive-ci`. It intentionally excludes most Anneal source | |
| # files so ordinary PRs can reuse the archive built from `main`. | |
| - name: Restore Anneal main Nix binary cache | |
| id: restore_anneal_main_nix_cache | |
| uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: anneal/v2/target/nix-cache-main | |
| key: anneal-v2-main-nix-cache-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('anneal/v2/flake.nix', 'anneal/v2/flake.lock', 'anneal/v2/rewrite-lake-vendor.py', 'anneal/v2/prune-lake-cache.py') }} | |
| # Pull request caches are scoped by GitHub to the PR merge ref, so they | |
| # can speed up repeated pushes to the same PR without becoming visible to | |
| # `main` or to sibling PRs. Keep this separate from the trusted main cache | |
| # so untrusted PR output cannot publish a shared substituter. | |
| - name: Restore Anneal PR Nix binary cache | |
| id: restore_anneal_pr_nix_cache | |
| if: github.event_name == 'pull_request' | |
| uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: anneal/v2/target/nix-cache-pr | |
| key: anneal-v2-pr-nix-cache-v1-${{ runner.os }}-${{ runner.arch }}-pr-${{ github.event.pull_request.number }}-${{ hashFiles('anneal/v2/flake.nix', 'anneal/v2/flake.lock', 'anneal/v2/rewrite-lake-vendor.py', 'anneal/v2/prune-lake-cache.py') }} | |
| - name: Install Nix | |
| uses: DeterminateSystems/determinate-nix-action@4eea0b33e3d1f02ecfe37cf16e7204c424009606 # v3.21.0 | |
| # On Ubuntu 24.04 (currently `ubuntu-latest`), AppArmor restricts unprivileged user namespaces by default. | |
| # The Nix build sandbox runs `steam-run` (which uses `bubblewrap`/`bwrap`) during the `mathlib-cache-download` | |
| # phase to create an FHS environment. `bwrap` requires creating a user namespace to set up uid mappings, | |
| # which fails with "Permission denied" unless this restriction is temporarily disabled on the host. | |
| # | |
| # We temporarily disable it right before the `nix build` step and re-enable it immediately after | |
| # to maintain the principle of least privilege. | |
| # | |
| # FIXME(#3412): Deduplicate this with what's repeated below? | |
| - name: Enable unprivileged user namespaces (Ubuntu 24.04) | |
| run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 | |
| - name: Build Anneal toolchain archive | |
| run: | | |
| set -euo pipefail | |
| mkdir -p target | |
| nix_args=() | |
| for cache in target/nix-cache-main target/nix-cache-pr; do | |
| if [ -f "$cache/nix-cache-info" ]; then | |
| nix_args+=(--extra-substituters "file://$PWD/$cache/?trusted=1") | |
| fi | |
| done | |
| nix build "${nix_args[@]}" .#omnibus-archive-ci --out-link target/anneal-exocrate.tar.zst | |
| archive="$(readlink -f target/anneal-exocrate.tar.zst)" | |
| rm target/anneal-exocrate.tar.zst | |
| cp "$archive" target/anneal-exocrate.tar.zst | |
| archive_size="$(python3 -c 'import os, sys; print(os.path.getsize(sys.argv[1]))' target/anneal-exocrate.tar.zst)" | |
| max_github_release_asset_size=2147483647 | |
| printf 'Built anneal-exocrate.tar.zst (%s bytes)\n' "$archive_size" | |
| if [ "$archive_size" -gt "$max_github_release_asset_size" ]; then | |
| echo "::error::anneal-exocrate.tar.zst is ${archive_size} bytes, which exceeds GitHub's ${max_github_release_asset_size}-byte release asset limit." | |
| exit 1 | |
| fi | |
| nix build "${nix_args[@]}" .#omnibus-archive-layout-check --no-link | |
| working-directory: anneal/v2 | |
| # Re-enable the AppArmor namespace restriction to restore the runner host's default security posture. | |
| # `if: always()` ensures this cleanup step runs even if the Nix build fails. | |
| - name: Restore AppArmor restriction | |
| if: always() | |
| run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=1 | |
| # Populate the default-branch cache only from trusted `main` pushes. | |
| # GitHub's dependency-cache scoping makes entries saved from `main` | |
| # visible to PRs targeting `main`, while entries from PR merge refs are | |
| # isolated to that PR. Saving only here keeps the cache useful across PRs | |
| # without letting a PR publish an archive for unrelated runs. | |
| - name: Populate Anneal Nix binary cache | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.restore_anneal_main_nix_cache.outputs.cache-hit != 'true' | |
| run: | | |
| set -euo pipefail | |
| mkdir -p target/nix-cache-main | |
| # The archive is already zstd-compressed, so keep the Nix binary-cache | |
| # wrapper cheap; higher levels add CPU time for negligible size wins. | |
| nix copy .#omnibus-archive-ci \ | |
| --to "file://$PWD/target/nix-cache-main/?compression=zstd&compression-level=1&trusted=1" | |
| working-directory: anneal/v2 | |
| - name: Save Anneal Nix binary cache | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.restore_anneal_main_nix_cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: anneal/v2/target/nix-cache-main | |
| key: ${{ steps.restore_anneal_main_nix_cache.outputs.cache-primary-key }} | |
| - name: Populate Anneal PR Nix binary cache | |
| if: github.event_name == 'pull_request' && steps.restore_anneal_pr_nix_cache.outputs.cache-hit != 'true' | |
| run: | | |
| set -euo pipefail | |
| mkdir -p target/nix-cache-pr | |
| # PR-local caches optimize repeated pushes to the same PR. The | |
| # archive itself uses zstd level 6; keep this wrapper at level 1. | |
| nix copy .#omnibus-archive-ci \ | |
| --to "file://$PWD/target/nix-cache-pr/?compression=zstd&compression-level=1&trusted=1" | |
| working-directory: anneal/v2 | |
| - name: Save Anneal PR Nix binary cache | |
| if: github.event_name == 'pull_request' && steps.restore_anneal_pr_nix_cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 | |
| with: | |
| path: anneal/v2/target/nix-cache-pr | |
| key: ${{ steps.restore_anneal_pr_nix_cache.outputs.cache-primary-key }} | |
| - name: Upload Anneal toolchain archive | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: anneal-exocrate.tar.zst | |
| path: anneal/v2/target/anneal-exocrate.tar.zst | |
| if-no-files-found: error | |
| retention-days: 1 | |
| compression-level: 0 | |
| v2: | |
| name: Run V2 tests | |
| runs-on: ubuntu-latest | |
| # Depending on `v2_nix_cache` avoids duplicate Nix builds; this job only | |
| # downloads the exact archive that the builder job produced. | |
| needs: v2_nix_cache | |
| permissions: | |
| actions: read # Required to download the toolchain artifact. | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - name: Download Anneal toolchain archive | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: anneal-exocrate.tar.zst | |
| path: anneal/v2/target | |
| # FIXME: Pin this nightly to the same Rust date encoded in | |
| # anneal/v2/flake.nix, or derive it from the archive metadata, so v2 CI is | |
| # reproducible instead of following whatever nightly happens to be latest. | |
| - name: Install latest nightly Rust | |
| uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # zizmor: ignore[superfluous-actions] | |
| with: | |
| toolchain: nightly | |
| - name: Run V2 tests | |
| run: cargo test --workspace --all-features # include, e.g., tests that assume exocrate prebuilt | |
| working-directory: anneal/v2 | |
| # Used to signal to branch protections that all other jobs have succeeded. | |
| all-jobs-succeed: | |
| # WARNING: This name is load-bearing! It's how GitHub's settings UI | |
| # configures which jobs to block on. DO NOT change this name without | |
| # updating the settings UI to match. | |
| name: All checks succeeded (anneal.yml) | |
| # On failure, we run and unconditionally exit with a failing status code. | |
| # On success, this job is skipped. Jobs skipped using `if:` are considered | |
| # to have succeeded: | |
| # | |
| # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks | |
| if: failure() | |
| runs-on: ubuntu-latest | |
| needs: [static_checks, anneal_tests, verify_examples, v2_nix_cache, v2] | |
| steps: | |
| - name: Mark the job as failed | |
| run: exit 1 |