run_benches.yml

on:
  pull_request:
    types: [labeled, opened, reopened, synchronize]

name: Run and Cache Benchmarks
permissions:
  contents: "read"

jobs:
  run_benchmarks:
    if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        backend: ["postgres", "sqlite", "mysql"]
    steps:
      - name: Checkout sources
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          persist-credentials: false

      - name: Install postgres (Linux)
        if: matrix.backend == 'postgres'
        env:
          PG_VERSION: 16
        run: |
          sudo apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get --purge remove postgresql\* -y
          sudo apt-get install gnupg2 -y
          curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc|sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg
          sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
          sudo apt-get update
          sudo apt-get install -y libpq-dev postgresql-$PG_VERSION
          sudo tee /etc/postgresql/$PG_VERSION/main/pg_hba.conf <<'EOF'
          local   all             postgres                                peer
          local   all             all                                     peer
          host    all             all             127.0.0.1/32            trust
          host    all             all             ::1/128                 trust
          EOF
          sudo service postgresql start $PG_VERSION && sleep 3
          sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';"
          sudo service postgresql restart $PG_VERSION && sleep 3
          echo 'DATABASE_URL=postgres://postgres:postgres@localhost:5432/' >> $GITHUB_ENV

      - name: Install sqlite (Linux)
        if: matrix.backend == 'sqlite'
        run: |
          sudo apt-get update
          sudo apt-get install -y libsqlite3-dev
          echo 'DATABASE_URL=/tmp/test.db' >> $GITHUB_ENV

      - name: Install mysql (Linux)
        if: matrix.backend == 'mysql'
        run: |
          sudo systemctl start mysql.service
          sudo apt-get update
          sudo apt-get -y install libmysqlclient-dev
          mysql -e "create database diesel_test; create database diesel_unit_test; grant all on \`diesel_%\`.* to 'root'@'localhost';" -uroot -proot
          echo 'DATABASE_URL=mysql://root:root@localhost/diesel_test' >> $GITHUB_ENV

      - name: Install rust toolchain
        run: |
          rustup update stable

      - name: Install critcmp
        run: cargo +stable install critcmp

      - name: Benchmark PR ${{ matrix.backend }}
        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" -- --save-baseline changes > pr_${{ matrix.backend }}.txt

      - name: Upload PR ${{ matrix.backend }} Benchmark Results
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: pr_${{ matrix.backend }}.txt
          path: ./pr_${{ matrix.backend }}.txt

      - name: Copy results
        run: |
          mv diesel_bench/target/criterion /tmp

      - name: Checkout base branch
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          repository: ${{ github.event.pull_request.base.repo.full_name }}
          persist-credentials: false

      - name: Copy results back
        run: |
          mkdir diesel_bench/target -p
          mv /tmp/criterion diesel_bench/target/criterion

      - name: Benchmark base ${{ matrix.backend }}
        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" -- --save-baseline main > base_${{ matrix.backend }}.txt

      - name: Upload base ${{ matrix.backend }} Benchmark Results
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: base_${{ matrix.backend }}.txt
          path: ./base_${{ matrix.backend }}.txt

      - name: Upload GitHub Event
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: event_${{ matrix.backend }}.json
          path: ${{ github.event_path }}

      - name: Critcmp
        run: |
          cd diesel_bench
          critcmp --baselines
          critcmp main change
          echo "# ${{matrix.backend}}" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          critcmp main changes >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY

  track_benchmarks:
    needs: run_benchmarks
    runs-on: ubuntu-latest
    environment: Bencher
    strategy:
      fail-fast: false
      matrix:
        backend: ["postgres", "sqlite", "mysql"]
    env:
      BENCHER_PROJECT: diesel
      BENCHER_ADAPTER: rust_criterion
      BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }}
      PR_BENCHMARK_RESULTS: pr_${{ matrix.backend }}.txt
      BASE_BENCHMARK_RESULTS: base_${{ matrix.backend }}.txt
      GITHUB_EVENT: event_${{ matrix.backend }}.json
      # This is the percentage that the PR can be slower than the base benchmark
      # Adjust this value to lower to make the test more sensitive to changes
      # Adjust this value to higher to make the test less sensitive to changes
      # https://bencher.dev/docs/explanation/thresholds/#percentage-upper-boundary
      UPPER_BOUNDARY: 1.0
    steps:
      - name: Download PR Benchmark Results
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
        with:
          name: ${{ env.PR_BENCHMARK_RESULTS }}
      - name: Download Base Benchmark Results
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
        with:
          name: ${{ env.BASE_BENCHMARK_RESULTS }}
      - name: Download GitHub Event
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
        with:
          name: ${{ env.GITHUB_EVENT }}
      - name: Export GitHub Event Data
        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
        with:
          script: |
            let fs = require('fs');
            let githubEvent = JSON.parse(fs.readFileSync("event.json", {encoding: 'utf8'}));
            console.log(githubEvent);
            core.exportVariable("PR_HEAD", githubEvent.pull_request.head.ref);
            core.exportVariable("PR_HEAD_SHA", githubEvent.pull_request.head.sha);
            core.exportVariable("PR_BASE", githubEvent.pull_request.base.ref);
            core.exportVariable("PR_BASE_SHA", githubEvent.pull_request.base.sha);
            core.exportVariable("PR_NUMBER", githubEvent.number);
      - uses: bencherdev/bencher@2f1532643adc0e69e52acaec936d227ff14da24f # v0.5.9
      - name: Track PR Base Benchmarks
        run: |
          bencher run \
          --token "${{ secrets.BENCHER_API_TOKEN }}" \
          --branch "$PR_BASE" \
          --hash "$PR_BASE_SHA" \
          --start-point-reset \
          --file "$BASE_BENCHMARK_RESULTS"
      - name: Track PR Head Benchmarks
        run: |
          bencher run \
          --token "${{ secrets.BENCHER_API_TOKEN }}" \
          --branch "$PR_HEAD" \
          --hash "$PR_HEAD_SHA" \
          --start-point "$PR_BASE" \
          --start-point-hash "$PR_BASE_SHA" \
          --start-point-reset \
          --threshold-measure latency \
          --threshold-test percentage \
          --threshold-upper-boundary ${{ env.UPPER_BOUNDARY }} \
          --thresholds-reset \
          --github-actions "${{ secrets.GITHUB_TOKEN }}" \
          --ci-number "$PR_NUMBER" \
          --err \
          --file "$PR_BENCHMARK_RESULTS"