nightly-perf.yml

# NOTE: All third-party actions are pinned by full commit SHA for supply-chain safety.
# To update an action: find the new version's commit SHA on GitHub (Tags → verify commit),
# replace the SHA, and keep the "# vX" comment in sync.
name: Nightly Performance Tests

on:
  schedule:
    # Run daily at 3:00 AM UTC
    - cron: '0 3 * * *'
  workflow_dispatch:

permissions:
  contents: read
  issues: write

jobs:
  performance-tests:
    name: Performance Benchmarks
    runs-on: macos-26
    timeout-minutes: 60
    concurrency:
      group: nightly-perf
      cancel-in-progress: false
    steps:
      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0

      - name: Select Xcode
        uses: ./.github/actions/select-xcode

      - name: Download Metal Toolchain
        run: |
          xcodebuild -downloadComponent MetalToolchain

      - name: Cache SPM Dependencies
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          path: DerivedData/SourcePackages
          key: spm-${{ runner.os }}-${{ hashFiles('Pine.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}
          restore-keys: |
            spm-${{ runner.os }}-

      - name: Build for Testing
        run: |
          xcodebuild build-for-testing \
            -skipPackagePluginValidation \
            -project Pine.xcodeproj \
            -scheme Pine \
            -destination "platform=macOS" \
            -derivedDataPath DerivedData \
            -only-testing:PinePerformanceTests \
            CODE_SIGN_IDENTITY=- \
            CODE_SIGNING_ALLOWED=NO

      - name: Run Performance Tests
        run: |
          xcodebuild test-without-building \
            -skipPackagePluginValidation \
            -project Pine.xcodeproj \
            -scheme Pine \
            -destination "platform=macOS" \
            -derivedDataPath DerivedData \
            -only-testing:PinePerformanceTests \
            -resultBundlePath PerformanceResults.xcresult \
            CODE_SIGN_IDENTITY=- \
            CODE_SIGNING_ALLOWED=NO

      - name: Performance Summary
        if: success() || failure()
        run: |
          XCRESULT="PerformanceResults.xcresult"
          if [ ! -d "$XCRESULT" ]; then
            echo "No .xcresult bundle found"
            exit 0
          fi

          echo "## Nightly Performance Test Results" >> "$GITHUB_STEP_SUMMARY"
          echo "" >> "$GITHUB_STEP_SUMMARY"
          echo "**Date:** $(date -u '+%Y-%m-%d %H:%M UTC')" >> "$GITHUB_STEP_SUMMARY"
          echo "" >> "$GITHUB_STEP_SUMMARY"

          # Extract test results using jq
          xcrun xcresulttool get test-results summary \
            --path "$XCRESULT" \
            --compact 2>/dev/null \
          | jq -r '
            "| Metric | Value |",
            "|--------|-------|",
            "| Total tests | \(.passedTests + .failedTests) |",
            "| Passed | \(.passedTests) |",
            "| Failed | \(.failedTests) |"
          ' >> "$GITHUB_STEP_SUMMARY" || echo "Could not extract test summary" >> "$GITHUB_STEP_SUMMARY"

          echo "" >> "$GITHUB_STEP_SUMMARY"
          echo "_Download the PerformanceResults.xcresult artifact and open in Xcode Test Navigator for detailed measure {} timings._" >> "$GITHUB_STEP_SUMMARY"

      - name: Check for Regressions
        id: regression-check
        if: success() || failure()
        run: |
          XCRESULT="PerformanceResults.xcresult"
          BASELINES="PinePerformanceTests/baselines.json"

          if [ ! -d "$XCRESULT" ] || [ ! -f "$BASELINES" ]; then
            echo "Skipping regression check — missing xcresult or baselines"
            echo "has_regression=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          set +e
          REPORT=$(python3 .github/scripts/check_perf_regression.py "$BASELINES" "$XCRESULT" 2>/tmp/perf_stderr.log)
          EXIT_CODE=$?
          set -e

          if [ -s /tmp/perf_stderr.log ]; then
            echo "::warning::check_perf_regression.py stderr:"
            cat /tmp/perf_stderr.log
          fi

          echo "$REPORT" >> "$GITHUB_STEP_SUMMARY"

          # Save report to a temp file for the issue body
          echo "$REPORT" > /tmp/regression_report.md

          if [ $EXIT_CODE -ne 0 ]; then
            echo "has_regression=true" >> "$GITHUB_OUTPUT"
          else
            echo "has_regression=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Upload Performance Results
        if: success() || failure()
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: PerformanceResults-${{ github.run_id }}
          path: PerformanceResults.xcresult
          retention-days: 30

      - name: Create Issue on Failure
        if: failure()
        env:
          GH_TOKEN: ${{ github.token }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          gh issue create \
            --title "Nightly performance tests failed ($(date -u '+%Y-%m-%d'))" \
            --body "## Summary

          The nightly performance test run failed.

          **Run:** $RUN_URL

          ## Next steps

          1. Check the workflow logs for failing test(s)
          2. Download the \`PerformanceResults.xcresult\` artifact for detailed timings
          3. Investigate whether this is a flaky test or a real regression" \
            --label "bug,testing"

      - name: Create Issue on Regression
        if: steps.regression-check.outputs.has_regression == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          REGRESSION_REPORT=$(cat /tmp/regression_report.md 2>/dev/null || echo "Report unavailable")
          BODY=$(cat <<ISSUE_EOF
          ## Summary

          Performance regression detected in nightly benchmarks.

          **Run:** $RUN_URL

          $REGRESSION_REPORT

          ## Next steps

          1. Download the \`PerformanceResults.xcresult\` artifact for detailed timings
          2. Check recent commits for performance-impacting changes
          3. If the regression is expected, update \`PinePerformanceTests/baselines.json\`
          ISSUE_EOF
          )

          # Check for existing open regression issue to avoid duplicates
          EXISTING=$(gh issue list --label "performance" --state open --search "Performance regression detected" --json number --jq '.[0].number' 2>/dev/null || true)

          if [ -n "$EXISTING" ] && [ "$EXISTING" != "null" ]; then
            echo "Adding comment to existing issue #$EXISTING"
            gh issue comment "$EXISTING" --body "$BODY"
          else
            gh issue create \
              --title "Performance regression detected ($(date -u '+%Y-%m-%d'))" \
              --body "$BODY" \
              --label "bug,performance,testing"
          fi