Nightly Performance Tests #128
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # NOTE: All third-party actions are pinned by full commit SHA for supply-chain safety. | |
| # To update an action: find the new version's commit SHA on GitHub (Tags → verify commit), | |
| # replace the SHA, and keep the "# vX" comment in sync. | |
| name: Nightly Performance Tests | |
| on: | |
| schedule: | |
| # Run daily at 3:00 AM UTC | |
| - cron: '0 3 * * *' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| issues: write | |
| jobs: | |
| performance-tests: | |
| name: Performance Benchmarks | |
| runs-on: macos-26 | |
| timeout-minutes: 60 | |
| concurrency: | |
| group: nightly-perf | |
| cancel-in-progress: false | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| - name: Select Xcode | |
| uses: ./.github/actions/select-xcode | |
| - name: Download Metal Toolchain | |
| run: | | |
| xcodebuild -downloadComponent MetalToolchain | |
| - name: Cache SPM Dependencies | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: DerivedData/SourcePackages | |
| key: spm-${{ runner.os }}-${{ hashFiles('Pine.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }} | |
| restore-keys: | | |
| spm-${{ runner.os }}- | |
| - name: Build for Testing | |
| run: | | |
| xcodebuild build-for-testing \ | |
| -skipPackagePluginValidation \ | |
| -project Pine.xcodeproj \ | |
| -scheme Pine \ | |
| -destination "platform=macOS" \ | |
| -derivedDataPath DerivedData \ | |
| -only-testing:PinePerformanceTests \ | |
| CODE_SIGN_IDENTITY=- \ | |
| CODE_SIGNING_ALLOWED=NO | |
| - name: Run Performance Tests | |
| run: | | |
| xcodebuild test-without-building \ | |
| -skipPackagePluginValidation \ | |
| -project Pine.xcodeproj \ | |
| -scheme Pine \ | |
| -destination "platform=macOS" \ | |
| -derivedDataPath DerivedData \ | |
| -only-testing:PinePerformanceTests \ | |
| -resultBundlePath PerformanceResults.xcresult \ | |
| CODE_SIGN_IDENTITY=- \ | |
| CODE_SIGNING_ALLOWED=NO | |
| - name: Performance Summary | |
| if: success() || failure() | |
| run: | | |
| XCRESULT="PerformanceResults.xcresult" | |
| if [ ! -d "$XCRESULT" ]; then | |
| echo "No .xcresult bundle found" | |
| exit 0 | |
| fi | |
| echo "## Nightly Performance Test Results" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Date:** $(date -u '+%Y-%m-%d %H:%M UTC')" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| # Extract test results using jq | |
| xcrun xcresulttool get test-results summary \ | |
| --path "$XCRESULT" \ | |
| --compact 2>/dev/null \ | |
| | jq -r ' | |
| "| Metric | Value |", | |
| "|--------|-------|", | |
| "| Total tests | \(.passedTests + .failedTests) |", | |
| "| Passed | \(.passedTests) |", | |
| "| Failed | \(.failedTests) |" | |
| ' >> "$GITHUB_STEP_SUMMARY" || echo "Could not extract test summary" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "_Download the PerformanceResults.xcresult artifact and open in Xcode Test Navigator for detailed measure {} timings._" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Check for Regressions | |
| id: regression-check | |
| if: success() || failure() | |
| run: | | |
| XCRESULT="PerformanceResults.xcresult" | |
| BASELINES="PinePerformanceTests/baselines.json" | |
| if [ ! -d "$XCRESULT" ] || [ ! -f "$BASELINES" ]; then | |
| echo "Skipping regression check — missing xcresult or baselines" | |
| echo "has_regression=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| set +e | |
| REPORT=$(python3 .github/scripts/check_perf_regression.py "$BASELINES" "$XCRESULT" 2>/tmp/perf_stderr.log) | |
| EXIT_CODE=$? | |
| set -e | |
| if [ -s /tmp/perf_stderr.log ]; then | |
| echo "::warning::check_perf_regression.py stderr:" | |
| cat /tmp/perf_stderr.log | |
| fi | |
| echo "$REPORT" >> "$GITHUB_STEP_SUMMARY" | |
| # Save report to a temp file for the issue body | |
| echo "$REPORT" > /tmp/regression_report.md | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "has_regression=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_regression=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Upload Performance Results | |
| if: success() || failure() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: PerformanceResults-${{ github.run_id }} | |
| path: PerformanceResults.xcresult | |
| retention-days: 30 | |
| - name: Create Issue on Failure | |
| if: failure() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| gh issue create \ | |
| --title "Nightly performance tests failed ($(date -u '+%Y-%m-%d'))" \ | |
| --body "## Summary | |
| The nightly performance test run failed. | |
| **Run:** $RUN_URL | |
| ## Next steps | |
| 1. Check the workflow logs for failing test(s) | |
| 2. Download the \`PerformanceResults.xcresult\` artifact for detailed timings | |
| 3. Investigate whether this is a flaky test or a real regression" \ | |
| --label "bug,testing" | |
| - name: Create Issue on Regression | |
| if: steps.regression-check.outputs.has_regression == 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| REGRESSION_REPORT=$(cat /tmp/regression_report.md 2>/dev/null || echo "Report unavailable") | |
| BODY=$(cat <<ISSUE_EOF | |
| ## Summary | |
| Performance regression detected in nightly benchmarks. | |
| **Run:** $RUN_URL | |
| $REGRESSION_REPORT | |
| ## Next steps | |
| 1. Download the \`PerformanceResults.xcresult\` artifact for detailed timings | |
| 2. Check recent commits for performance-impacting changes | |
| 3. If the regression is expected, update \`PinePerformanceTests/baselines.json\` | |
| ISSUE_EOF | |
| ) | |
| # Check for existing open regression issue to avoid duplicates | |
| EXISTING=$(gh issue list --label "performance" --state open --search "Performance regression detected" --json number --jq '.[0].number' 2>/dev/null || true) | |
| if [ -n "$EXISTING" ] && [ "$EXISTING" != "null" ]; then | |
| echo "Adding comment to existing issue #$EXISTING" | |
| gh issue comment "$EXISTING" --body "$BODY" | |
| else | |
| gh issue create \ | |
| --title "Performance regression detected ($(date -u '+%Y-%m-%d'))" \ | |
| --body "$BODY" \ | |
| --label "bug,performance,testing" | |
| fi |