Add performance benchmark comparison workflow for pull requests #3
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Comparison for Pull Requests | |
| on: | |
| pull_request: | |
| branches: [master] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| benchmark-pr: | |
| name: Performance benchmark comparison | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up JDK 1.8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '8' | |
| distribution: 'temurin' | |
| # Save commit SHAs for display | |
| - name: Save commit info | |
| id: commits | |
| run: | | |
| BASE_SHA="${{ github.event.pull_request.base.sha }}" | |
| HEAD_SHA="${{ github.event.pull_request.head.sha }}" | |
| echo "base_short=${BASE_SHA:0:7}" >> $GITHUB_OUTPUT | |
| echo "head_short=${HEAD_SHA:0:7}" >> $GITHUB_OUTPUT | |
| # Run benchmark on PR branch (with reduced iterations for faster feedback) | |
| - name: Run benchmark on PR branch | |
| run: | | |
| mvn clean compile test-compile | |
| # Run JMH with reduced iterations for faster PR feedback | |
| # Uses: 2 warmup iterations, 3 measurement iterations instead of default 5,5 | |
| # Pattern targets only EnforcerBenchmarkTest, not CachedEnforcerBenchmarkTest | |
| java -cp "target/test-classes:target/classes:$(mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=/dev/stdout -q)" \ | |
| org.openjdk.jmh.Main "^.*\\.EnforcerBenchmarkTest\\." -wi 2 -i 3 -f 1 -r 1 -w 1 -rf json -rff pr-results.json 2>&1 | tee pr-bench.txt | |
| # Checkout base branch and run benchmark | |
| - name: Checkout base branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| clean: false | |
| path: base | |
| # Run benchmark on base branch (with reduced iterations for faster feedback) | |
| - name: Run benchmark on base branch | |
| working-directory: base | |
| run: | | |
| mvn clean compile test-compile | |
| # Run JMH with reduced iterations for faster PR feedback | |
| # Uses: 2 warmup iterations, 3 measurement iterations instead of default 5,5 | |
| # Pattern targets only EnforcerBenchmarkTest, not CachedEnforcerBenchmarkTest | |
| java -cp "target/test-classes:target/classes:$(mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=/dev/stdout -q)" \ | |
| org.openjdk.jmh.Main "^.*\\.EnforcerBenchmarkTest\\." -wi 2 -i 3 -f 1 -r 1 -w 1 -rf json -rff ../base-results.json 2>&1 | tee ../base-bench.txt | |
| # Compare benchmarks | |
| - name: Parse and compare benchmarks | |
| id: compare | |
| run: | | |
| cat > compare_benchmarks.py << 'PYTHON_SCRIPT' | |
| import re | |
| import sys | |
| import json | |
| import os | |
| def parse_jmh_json(filename): | |
| """Parse JMH JSON output and extract throughput results.""" | |
| if not os.path.exists(filename): | |
| return {} | |
| results = {} | |
| try: | |
| with open(filename, 'r') as f: | |
| data = json.load(f) | |
| for benchmark in data: | |
| benchmark_name = benchmark.get('benchmark', '') | |
| # Extract just the method name | |
| method_name = benchmark_name.split('.')[-1] if '.' in benchmark_name else benchmark_name | |
| # Get primary metric (throughput) | |
| primary_metric = benchmark.get('primaryMetric', {}) | |
| score = primary_metric.get('score', 0) | |
| error = primary_metric.get('scoreError', 0) | |
| if score > 0: | |
| results[method_name] = {'score': score, 'error': error} | |
| except (json.JSONDecodeError, KeyError, ValueError) as e: | |
| print(f"Error parsing JSON {filename}: {e}", file=sys.stderr) | |
| return results | |
| def parse_jmh_output(filename): | |
| """Parse JMH benchmark output and extract throughput results.""" | |
| results = {} | |
| if not os.path.exists(filename): | |
| return {} | |
| try: | |
| with open(filename, 'r') as f: | |
| content = f.read() | |
| except (IOError, OSError) as e: | |
| print(f"Error reading file {filename}: {e}", file=sys.stderr) | |
| return {} | |
| # Match benchmark results with throughput mode | |
| # Format: Benchmark Mode Cnt Score Error Units | |
| # org.casbin...benchmarkXXX thrpt 5 12345.678 ± 123.456 ops/ms | |
| # Using flexible pattern for plus-minus symbol to handle different encodings | |
| pattern = r'(\S+)\s+thrpt\s+\d+\s+([\d.]+)\s+[±+\-]\s+([\d.]+)\s+ops/ms' | |
| for match in re.finditer(pattern, content): | |
| benchmark_name = match.group(1) | |
| score = float(match.group(2)) | |
| error = float(match.group(3)) | |
| # Extract just the method name | |
| method_name = benchmark_name.split('.')[-1] if '.' in benchmark_name else benchmark_name | |
| results[method_name] = {'score': score, 'error': error} | |
| return results | |
| def format_comparison(base_results, pr_results): | |
| """Format benchmark comparison as markdown table.""" | |
| if not base_results and not pr_results: | |
| return "⚠️ No benchmark results found in either base or PR branch." | |
| if not base_results: | |
| return "⚠️ No benchmark results found in base branch." | |
| if not pr_results: | |
| return "⚠️ No benchmark results found in PR branch." | |
| # Combine all benchmark names | |
| all_benchmarks = sorted(set(list(base_results.keys()) + list(pr_results.keys()))) | |
| lines = [] | |
| lines.append("| Benchmark | Base (ops/ms) | PR (ops/ms) | Change | Status |") | |
| lines.append("|-----------|---------------|-------------|--------|--------|") | |
| for benchmark in all_benchmarks: | |
| base = base_results.get(benchmark) | |
| pr = pr_results.get(benchmark) | |
| if base and pr: | |
| base_score = base['score'] | |
| pr_score = pr['score'] | |
| # Calculate percentage change | |
| if base_score > 0: | |
| change_pct = ((pr_score - base_score) / base_score) * 100 | |
| change_str = f"{change_pct:+.2f}%" | |
| # Determine status emoji | |
| if change_pct > 5: | |
| status = "🚀 Faster" | |
| elif change_pct < -5: | |
| status = "🐌 Slower" | |
| else: | |
| status = "➡️ Similar" | |
| else: | |
| change_str = "N/A" | |
| status = "⚠️" | |
| base_str = f"{base_score:.2f} ± {base['error']:.2f}" | |
| pr_str = f"{pr_score:.2f} ± {pr['error']:.2f}" | |
| elif base: | |
| base_str = f"{base['score']:.2f} ± {base['error']:.2f}" | |
| pr_str = "❌ Missing" | |
| change_str = "N/A" | |
| status = "⚠️" | |
| else: | |
| base_str = "❌ Missing" | |
| pr_str = f"{pr['score']:.2f} ± {pr['error']:.2f}" | |
| change_str = "N/A" | |
| status = "🆕 New" | |
| lines.append(f"| {benchmark} | {base_str} | {pr_str} | {change_str} | {status} |") | |
| return '\n'.join(lines) | |
| # Main execution - try JSON first, fallback to text parsing | |
| base_results = parse_jmh_json('base-results.json') | |
| if not base_results: | |
| base_results = parse_jmh_output('base-bench.txt') | |
| pr_results = parse_jmh_json('pr-results.json') | |
| if not pr_results: | |
| pr_results = parse_jmh_output('pr-bench.txt') | |
| comparison_table = format_comparison(base_results, pr_results) | |
| # Save to file for GitHub comment | |
| with open('comparison.md', 'w') as f: | |
| f.write(comparison_table) | |
| print(comparison_table) | |
| PYTHON_SCRIPT | |
| python3 compare_benchmarks.py | |
| # Post comment with results | |
| - name: Post benchmark comparison comment | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| let comparison = ''; | |
| try { | |
| comparison = fs.readFileSync('comparison.md', 'utf8'); | |
| } catch (error) { | |
| comparison = '⚠️ Failed to read comparison results.'; | |
| } | |
| const commentBody = `## 📊 Benchmark Comparison | |
| Comparing base branch (\`${{ steps.commits.outputs.base_short }}\`) vs PR branch (\`${{ steps.commits.outputs.head_short }}\`) | |
| ${comparison} | |
| <sub>🤖 This comment will be automatically updated with the latest benchmark results.</sub>`; | |
| // Find existing comment | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const botComment = comments.find(comment => | |
| comment.user.type === 'Bot' && | |
| comment.body.includes('📊 Benchmark Comparison') | |
| ); | |
| if (botComment) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: botComment.id, | |
| body: commentBody | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: commentBody | |
| }); | |
| } |