Add --repeat to examples #111
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| workflow_dispatch: | |
| # TODO: Remove pull_request trigger after testing | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| push: | |
| branches: [main] | |
| permissions: | |
| id-token: write # Required for OIDC authentication to AWS | |
| contents: read | |
| env: | |
| BENCHMARK_REPEAT_COUNT: 5 | |
| jobs: | |
| benchmarks: | |
| name: benchmarks-${{ matrix.name }} | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runner: 'c7i-16xlarge' | |
| name: 'x86-intel-64cpu-128gb' | |
| - runner: 'c8g-16xlarge' | |
| name: 'arm-graviton4-64cpu-128gb' | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref || github.ref }} | |
| - name: Set safe directory | |
| run: git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - name: Install deps | |
| run: | | |
| sudo yum -y install gcc openssl-devel curl jq z3-devel clang pkg-config python3 python3-pip | |
| pip3 install -r .github/scripts/requirements.txt | |
| - name: Setup Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: benchmarks-${{ matrix.name }} | |
| cache-on-failure: true | |
| - name: Run platform diagnostics | |
| run: cargo test -p binius-utils --features platform-diagnostics test_platform_diagnostics -- --nocapture | |
| env: | |
| RUSTFLAGS: "-C target-cpu=native" | |
| - name: Run benchmarks | |
| run: | | |
| python3 .github/scripts/run_benchmarks.py --repeat ${{ env.BENCHMARK_REPEAT_COUNT }} | |
| env: | |
| PERFETTO_PLATFORM_NAME: "${{ matrix.runner }}" | |
| RUSTFLAGS: "-C target-cpu=native" | |
| - name: List perfetto traces and summaries | |
| if: always() | |
| run: | | |
| echo "=== Perfetto traces directory structure ===" | |
| if [ -d perfetto_traces ]; then | |
| find perfetto_traces -type f | sort | |
| else | |
| echo "DEBUG: No perfetto_traces directory found" | |
| fi | |
| echo "" | |
| echo "=== Benchmark summaries ===" | |
| if [ -d benchmark_summaries ]; then | |
| find benchmark_summaries -type f | sort | |
| else | |
| echo "DEBUG: No benchmark_summaries directory found" | |
| fi | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: ${{ secrets.AWS_UPLOAD_ROLE }} | |
| aws-region: us-east-1 | |
| - name: Upload perfetto traces to S3 | |
| run: | | |
| BRANCH_NAME="${{ github.head_ref || github.ref_name }}" | |
| echo "DEBUG: Branch name: $BRANCH_NAME" | |
| if [ "$BRANCH_NAME" = "main" ]; then | |
| BRANCH_PATH="main" | |
| else | |
| # Sanitize branch name | |
| # Lowercase, keep a-z 0-9 . _ -, replace others with dash, remove leading/trailing dashes | |
| SANITIZED_BRANCH=$(echo "$BRANCH_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g' | sed 's/^-*//g' | sed 's/-*$//g' | sed 's/--*/-/g') | |
| BRANCH_PATH="branch-${SANITIZED_BRANCH}" | |
| fi | |
| echo "DEBUG: Branch path: $BRANCH_PATH" | |
| if [ -d perfetto_traces ]; then | |
| echo "DEBUG: Uploading perfetto traces to S3.." | |
| aws s3 cp "perfetto_traces/" "${{ secrets.PERFETTO_BUCKET }}/traces/binius64/${BRANCH_PATH}/" --recursive | |
| echo "DEBUG: Traces uploaded to: ${{ secrets.PERFETTO_BUCKET }}/traces/binius64/${BRANCH_PATH}/" | |
| else | |
| echo "DEBUG: No perfetto_traces directory to upload" | |
| fi | |
| echo "BRANCH_PATH=${BRANCH_PATH}" >> $GITHUB_ENV | |
| - name: Generate benchmark report for GitHub Summary | |
| if: always() | |
| run: | | |
| echo "DEBUG: Generating benchmark report..." | |
| echo "DEBUG: BRANCH_PATH=${BRANCH_PATH}" | |
| echo "DEBUG: Current directory contents:" | |
| ls -la | |
| echo "DEBUG: Running report generation script..." | |
| python3 .github/scripts/generate_benchmark_report.py \ | |
| --branch-path "${BRANCH_PATH}" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| echo "DEBUG: Report generation completed, exit code: $?" | |
| - name: Upload benchmark summaries as artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-summaries-${{ matrix.name }} | |
| path: | | |
| benchmark_summaries/*.json | |
| perfetto_traces/**/metrics.json | |
| retention-days: 30 | |
| publish-benchmark-results: | |
| name: Publish benchmark results to binius-benchmark repo | |
| # TODO: Re-enable condition | |
| # if: github.ref == 'refs/heads/main' | |
| needs: [benchmarks] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Download all benchmark artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ./artifacts | |
| pattern: benchmark-summaries-* | |
| merge-multiple: true | |
| # GitHub App authentication | |
| - name: Create GitHub App token | |
| uses: actions/create-github-app-token@v2 | |
| id: app-token | |
| with: | |
| app-id: ${{ vars.BENCHMARK_PUBLISHER_APP_ID }} | |
| private-key: ${{ secrets.BENCHMARK_PUBLISHER_PRIVATE_KEY }} | |
| owner: IrreducibleOSS | |
| repositories: binius-benchmark | |
| - name: Get GitHub App User ID | |
| id: get-user-id | |
| run: echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT" | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| - name: Configure git credentials globally | |
| run: | | |
| git config --global user.name '${{ steps.app-token.outputs.app-slug }}[bot]' | |
| git config --global user.email '${{ steps.get-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com' | |
| - name: Publish benchmark results with retry | |
| uses: nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 5 | |
| retry_wait_seconds: 30 | |
| command: | | |
| set -e # Exit on any error to trigger retry | |
| # Clone and setup | |
| rm -rf /tmp/binius-benchmark | |
| git clone "https://x-access-token:${GH_TOKEN}@github.com/IrreducibleOSS/binius-benchmark.git" /tmp/binius-benchmark | |
| cd /tmp/binius-benchmark | |
| git pull origin main | |
| # Process all benchmark-results files | |
| echo "Processing benchmark results..." | |
| python3 "${{ env.BENCHMARK_PROCESSOR_SCRIPT }}" "$GITHUB_WORKSPACE"/artifacts/benchmark_summaries/benchmark-results-*.json | |
| # Commit if changes exist | |
| git add . | |
| if ! git diff --staged --quiet; then | |
| git commit -m "Update benchmark data - binius64@${{ github.sha }}" | |
| git push origin main | |
| echo "🎉 Successfully pushed changes" | |
| else | |
| echo "ℹ️ No changes to commit" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ steps.app-token.outputs.token }} | |
| BENCHMARK_PROCESSOR_SCRIPT: scripts/update.py |