performance.yml

name: Performance comparison

on:
  workflow_dispatch:
    inputs:
      baseline:
        description: "Baseline ref (use a previous release tag for release review)"
        type: string
        default: main
        required: true
  release:
    types: [published]

permissions:
  contents: read

defaults:
  run:
    shell: bash

jobs:
  compare:
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    env:
      ERL_FLAGS: "+S 4:4"
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: candidate
          fetch-depth: 0
          persist-credentials: false
      - name: Select baseline
        id: baseline
        env:
          BASELINE: ${{ inputs.baseline }}
        run: |
          if [ -z "$BASELINE" ]; then
            BASELINE=$(git -C candidate describe --tags --abbrev=0 "${GITHUB_SHA}^")
          fi
          printf 'ref=%s\n' "$BASELINE" >> "$GITHUB_OUTPUT"
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: baseline
          ref: ${{ steps.baseline.outputs.ref }}
          persist-credentials: false
      - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
        with:
          elixir-version: "> 0"
          otp-version: "> 0"
      - name: Record environment and revisions
        run: |
          mkdir -p reports
          {
            elixir --version
            uname -a
            lscpu
            git -C baseline rev-parse HEAD
            git -C candidate rev-parse HEAD
          } | tee reports/environment.log
      - name: Benchee baseline and candidate on the same runner
        run: |
          GROUP_BENCH_ROOT="$PWD/baseline" \
          GROUP_BENCH_OUTPUT="$PWD/reports/baseline.benchee" \
          GROUP_BENCH_TAG=baseline \
            elixir candidate/priv/bench/comparison.exs 2>&1 | tee reports/benchee-baseline.log
          GROUP_BENCH_ROOT="$PWD/candidate" \
          GROUP_BENCH_OUTPUT="$PWD/reports/candidate.benchee" \
          GROUP_BENCH_BASELINE="$PWD/reports/baseline.benchee" \
            elixir candidate/priv/bench/comparison.exs 2>&1 | tee reports/benchee-candidate.log
      - name: Distributed load and recovery, baseline then candidate
        run: |
          epmd -daemon
          for revision in baseline candidate; do
            timeout 20m bash "$revision/priv/bench/run_distributed.sh" 2>&1 |
              tee "reports/distributed-$revision.log"
          done
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: performance-comparison
          path: reports
          retention-days: 30