benchmarks.yaml

name: Benchmarks

on:
  # Manual trigger with options
  workflow_dispatch:
    inputs:
      use_hetzner:
        description: 'Use Hetzner Cloud for benchmarks'
        required: false
        default: 'true'
        type: choice
        options:
          - 'true'
          - 'false'
      branch:
        description: 'Branch to checkout before running benchmarks'
        required: false
        default: 'main'
        type: string
      python_version:
        description: 'Python version to use for benchmarks'
        required: false
        default: '3.11'
        type: choice
        options:
          - '3.11'
          - '3.12'
          - '3.13'
      server_type:
        description: 'Hetzner server type'
        required: false
        default: 'cx23'
        type: choice
        options:
          - 'cx23'   # 2 vCPU, 4GB RAM (shared)
          - 'cx33'   # 4 vCPU, 8GB RAM (shared)
          - 'cx43'   # 8 vCPU, 16GB RAM (shared)
          - 'cx53'   # 16 vCPU, 32GB RAM (shared)
          - 'ccx13'  # 2 vCPU, 8GB RAM (dedicated AMD EPYC)
          - 'ccx23'  # 4 vCPU, 16GB RAM (dedicated AMD EPYC)
          - 'ccx33'  # 8 vCPU, 32GB RAM (dedicated AMD EPYC)
          - 'ccx43'  # 16 vCPU, 64GB RAM (dedicated AMD EPYC)
          - 'ccx53'  # 32 vCPU, 128GB RAM (dedicated AMD EPYC)
          - 'ccx63'  # 48 vCPU, 192GB RAM (dedicated AMD EPYC)

  # Run on pushes to main (for tracking performance over time)
  push:
    branches:
      - main
    paths:
      - 'csp_benchmarks/benchmarks/**'
      - 'csp_benchmarks/asv.conf.json'

concurrency:
  group: benchmarks-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: write
  pull-requests: write

jobs:
  # Quick local benchmarks for PRs
  benchmark-local:
    if: github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && github.event.inputs.use_hetzner == 'false')
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
          ref: ${{ github.event.inputs.branch || github.ref }}

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ github.event.inputs.python_version || '3.11' }}

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential cmake ninja-build libboost-all-dev

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -e ".[develop]"

      - name: Configure ASV machine
        run: |
          cp csp_benchmarks/asv-machine.json ~/.asv-machine.json
          make benchmark-init

      - name: Run quick benchmarks
        run: make benchmark-quick MACHINE=github-actions

      - name: Compare with main
        if: github.event_name == 'pull_request'
        run: |
          python -m asv continuous --config csp_benchmarks/asv.conf.json --verbose main HEAD || true

      - name: Generate HTML report
        run: make benchmark-publish

      - name: Upload benchmark results
        uses: actions/upload-artifact@v7
        with:
          name: benchmark-results
          path: |
            csp_benchmarks/results/
            .asv/html/

  # Full benchmarks on Hetzner Cloud
  benchmark-hetzner:
    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.use_hetzner == 'true') || (github.event_name == 'push' && github.ref == 'refs/heads/main')
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
          ref: ${{ github.event.inputs.branch || github.ref }}

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ github.event.inputs.python_version || '3.11' }}

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -e ".[develop,hetzner]"

      - name: Set up SSH key
        env:
          SSH_PRIVATE_KEY: ${{ secrets.HETZNER_SSH_PRIVATE_KEY }}
        run: |
          if [ -z "$SSH_PRIVATE_KEY" ]; then
            echo "ERROR: HETZNER_SSH_PRIVATE_KEY secret is not set!"
            echo "Please add an SSH private key to your repository secrets."
            echo "The corresponding public key must already be registered in Hetzner Cloud."
            exit 1
          fi
          mkdir -p ~/.ssh
          printf "%s\n" "$SSH_PRIVATE_KEY" > ~/.ssh/hetzner_key
          chmod 600 ~/.ssh/hetzner_key

      - name: Run benchmarks on Hetzner
        env:
          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          python -m csp_benchmarks.hetzner.cli run \
            --server-type ${{ github.event.inputs.server_type || 'cx23' }} \
            --ssh-key ~/.ssh/hetzner_key \
            --ssh-key-name benchmarks \
            --branch ${{ github.event.inputs.branch || 'main' }} \
            --python-version ${{ github.event.inputs.python_version || '3.11' }} \
            --push

      - name: Cleanup servers
        if: always()
        env:
          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
        run: |
          python -m csp_benchmarks.hetzner.cli cleanup || true

  # Publish results to GitHub Pages
  publish-results:
    needs: [benchmark-hetzner, benchmark-local]
    if: always() && (needs.benchmark-hetzner.result == 'success' || needs.benchmark-local.result == 'success')
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install asv

      - name: Pull latest results
        run: |
          git pull origin main || true

      - name: Check for results
        run: |
          echo "Current directory:"
          pwd
          echo "Looking for asv.conf.json:"
          ls -la csp_benchmarks/asv.conf.json || echo "Config not found"
          echo "Looking for results (relative to config file):"
          ls -la csp_benchmarks/results/ || echo "No results directory"
          echo "Sample result files:"
          ls csp_benchmarks/results/*/*.json 2>/dev/null | head -5 || echo "No result files found"

      - name: Generate HTML report
        run: |
          if [ -f "csp_benchmarks/asv.conf.json" ] && [ -d "csp_benchmarks/results" ]; then
            make benchmark-publish
          else
            echo "Skipping publish - no config or results found"
          fi

      - name: Deploy to GitHub Pages
        uses: peaceiris/actions-gh-pages@v4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: .asv/html