asv #645
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: asv | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: 0 4 * * * | |
| concurrency: | |
| group: "pages" | |
| cancel-in-progress: false | |
| env: | |
| # NOTE: For the first clean run, this should be a high-ish (~20) number to | |
| # populate the history. For all subsequent runs where data is cached in | |
| # `.asv/results/github-actions/`, it can be a lower number for better | |
| # runtimes in CI | |
| NUM_BENCHMARK_COMMITS: 5 | |
| BRANCH: main | |
| jobs: | |
| asv: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pages: write | |
| id-token: write | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| # NOTE: This only grabs the HEAD commit of the main branch, so we can't | |
| # use it for retrospective benchmarking. Instead, we manually | |
| # pull the submodule in the next step | |
| # with: | |
| # submodules: 'true' | |
| - name: Manually pull submodules | |
| run: | | |
| git config --global url."https://github.com/".insteadOf "git@github.com:" &&\ | |
| git submodule update --init &&\ | |
| cd xdsl &&\ | |
| git fetch --all &&\ | |
| git branch &&\ | |
| git checkout $BRANCH &&\ | |
| git pull | |
| - uses: astral-sh/setup-uv@v8.3.2 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "uv.lock" | |
| # Use `actions/setup-python@v5` rather than `uv python install 3.x` to | |
| # ensure that it is correctly in the $PATH for `asv` | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: | | |
| 3.11 | |
| pypy3.11 | |
| 3.13 | |
| - name: Check Python installations | |
| run: | | |
| which python3.11 && python3.11 --version | |
| which pypy.11 && pypy3.11 --version | |
| which python3.13 && python3.13 --version | |
| uv python list | |
| - name: Include existing artefacts | |
| run: rm -rf xdsl/.asv && mv .asv xdsl/.asv | |
| - name: Install the project | |
| working-directory: ./xdsl | |
| run: uv sync --group bench | |
| - name: Fingerprint the machine | |
| working-directory: ./xdsl | |
| run: | | |
| echo "OS: `uname -sr`" | |
| echo "Architecture: `uname -m`" | |
| echo "CPU: `lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1'`" | |
| echo "Cores: `nproc --all`" | |
| echo "RAM: `grep MemTotal /proc/meminfo | awk '{print $2}'`" | |
| uv run asv machine -v --yes \ | |
| --machine github-action \ | |
| --os "`uname -sr`" \ | |
| --arch "`uname -m`" \ | |
| --cpu "`lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1'`" \ | |
| --num_cpu `nproc --all` \ | |
| --ram `grep MemTotal /proc/meminfo | awk '{print $2}'` | |
| - name: Run the ASV benchmarks | |
| id: asv_run | |
| continue-on-error: true | |
| working-directory: ./xdsl | |
| run: | | |
| mkdir -p ../asv-logs | |
| set -o pipefail | |
| uv run asv run $BRANCH~$NUM_BENCHMARK_COMMITS..$BRANCH 2>&1 | tee ../asv-logs/asv-run.log | |
| - name: Rerun failed benchmarks with stderr | |
| if: steps.asv_run.outcome == 'failure' | |
| working-directory: ./xdsl | |
| run: | | |
| python - <<'PY' | |
| import pathlib | |
| import re | |
| log_path = pathlib.Path("../asv-logs/asv-run.log") | |
| bench_path = pathlib.Path("../asv-logs/asv-failed-benches.txt") | |
| failed = [] | |
| for line in log_path.read_text().splitlines(): | |
| match = re.search(r"([A-Za-z0-9_.]+)\s+failed$", line) | |
| if match: | |
| failed.append(match.group(1)) | |
| unique_failed = sorted(set(failed)) | |
| bench_path.write_text("\n".join(unique_failed) + ("\n" if unique_failed else "")) | |
| print(f"Found {len(unique_failed)} failed benchmarks.") | |
| PY | |
| if [ -s ../asv-logs/asv-failed-benches.txt ]; then | |
| while IFS= read -r bench; do | |
| echo "=== Rerun for $bench ===" | tee -a ../asv-logs/asv-rerun.log | |
| uv run asv run "$BRANCH~$NUM_BENCHMARK_COMMITS..$BRANCH" \ | |
| --bench "$bench" \ | |
| --quick \ | |
| --show-stderr \ | |
| --dry-run \ | |
| -v 2>&1 | tee -a ../asv-logs/asv-rerun.log | |
| done < ../asv-logs/asv-failed-benches.txt | |
| else | |
| echo "No failed benchmark names were parsed from asv-run.log" | tee -a ../asv-logs/asv-rerun.log | |
| fi | |
| - name: Upload ASV debug logs | |
| if: always() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f | |
| with: | |
| name: asv-debug-logs | |
| path: asv-logs/ | |
| - name: Fail job if ASV run failed | |
| if: steps.asv_run.outcome == 'failure' | |
| run: | | |
| echo "ASV benchmark run failed. See the asv-debug-logs artifact." >&2 | |
| exit 1 | |
| - name: Generate the ASV website | |
| working-directory: ./xdsl | |
| run: uv run asv publish | |
| - name: Copy back generated artefacts | |
| run: mv xdsl/.asv .asv | |
| - name: Commit the generated benchmark results | |
| uses: stefanzweifel/git-auto-commit-action@v7 | |
| with: | |
| commit_message: "Add CI benchmark runs" | |
| # CI commits should not be attributed to action author, to avoid extraneous activity | |
| commit_author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" | |
| - name: Setup Pages | |
| uses: actions/configure-pages@v6 | |
| - name: Upload artifact | |
| uses: actions/upload-pages-artifact@v5 | |
| with: | |
| path: .asv/html/ | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v5 | |
| notify-on-failure: | |
| needs: asv | |
| if: ${{ failure() }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Send Zulip notification | |
| uses: zulip/github-actions-zulip/send-message@v2 | |
| with: | |
| api-key: ${{ secrets.ZULIP_FAILURE_BOT_API_KEY }} | |
| email: ${{ secrets.ZULIP_FAILURE_BOT_EMAIL }} | |
| organization-url: "https://xdsl.zulipchat.com" | |
| to: "Framework" | |
| type: "stream" | |
| topic: "xdsl-bench Failure" | |
| content: | | |
| ❗ Workflow [${{ github.workflow }}](${{ | |
| github.server_url }}/${{ github.repository }}/actions/runs/${{ | |
| github.run_id }}) failed! |