catanatron/.github/workflows/perf-regression.yml at d2531c23a3d154a08fccfe05b0ba4d18ba75ac9e · bcollazo/catanatron · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
# This workflow will compare the current branch with the main branch and evaluate for performance differences.

name: Performance Regression Test

on:
  pull_request:
    branches: main

jobs:
  build-python-and-test-performance:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.12
        uses: actions/setup-python@v4
        with:
          python-version: 3.12
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip setuptools
          pip install .[web,gym,dev]

      - name: Checkout PR and Main Branch
        uses: actions/checkout@v4
        with:
          path: pr
      - uses: actions/checkout@v4
        with:
          ref: main
          path: main

      - name: Run Benchmark on Main (baseline)
        run: cd main && pytest tests/integration_tests/test_speed.py --benchmark-only --benchmark-json=benchmark.json
      - name: Run Benchmark on PR
        run: cd pr && pytest tests/integration_tests/test_speed.py --benchmark-only --benchmark-json=benchmark.json

      - name: Compare Results
        uses: openpgpjs/github-action-pull-request-benchmark@v1
        with:
          tool: "pytest"
          pr-benchmark-file-path: pr/benchmark.json
          base-benchmark-file-path: main/benchmark.json
          # A comment will be left on the latest PR commit if `alert-threshold` is exceeded
          comment-on-alert: true
          alert-threshold: "130%"
          # Workflow will fail if `fail-threshold` is exceeded
          fail-on-alert: false
          fail-threshold: "150%"
          # A token is needed to leave commit comments
          github-token: ${{ secrets.GITHUB_TOKEN }}