H-5331: Support CPU and Wall time profiling in benchmarks #12123
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bench | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: hashintel | |
| TURBO_CACHE: remote:rw | |
| VAULT_ADDR: ${{ secrets.VAULT_ADDR }} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| unit: ${{ steps.packages.outputs.unit }} | |
| integration: ${{ steps.packages.outputs.integration }} | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install tools | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| rust: false | |
| - name: Determine changed packages | |
| id: packages | |
| run: | | |
| UNIT_BENCH_QUERY='query { affectedPackages(base: "HEAD^", filter: {has: {field: TASK_NAME, value: "bench:unit"}}) { items { name path } } }' | |
| INTEGRATION_BENCH_QUERY='query { affectedPackages(base: "HEAD^", filter: {has: {field: TASK_NAME, value: "bench:integration"}}) { items { name path } } }' | |
| UNIT_BENCH_PACKAGES=$(turbo query "$UNIT_BENCH_QUERY" \ | |
| | jq --compact-output '.data.affectedPackages.items | [(.[] | select(.name != "//"))] | { name: [.[].name], include: . }') | |
| INTEGRATION_BENCH_PACKAGES=$(turbo query "$INTEGRATION_BENCH_QUERY" \ | |
| | jq --compact-output '.data.affectedPackages.items | [(.[] | select(.name != "//"))] | { name: [.[].name], include: . }') | |
| echo "unit=$UNIT_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT | |
| echo "integration=$INTEGRATION_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT | |
| unit-benches: | |
| name: Unit | |
| needs: [setup] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| matrix: ${{ fromJSON(needs.setup.outputs.unit) }} | |
| fail-fast: false | |
| if: needs.setup.outputs.unit != '{"name":[],"include":[]}' | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout base branch | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| - name: Install tools | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prune repository | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.name }} | |
| @rust/hash-repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Cache Rust dependencies | |
| if: github.event_name == 'pull_request' | |
| uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 | |
| with: | |
| workspaces: ${{ matrix.path }} | |
| save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }} | |
| - name: Run base benches | |
| if: github.event_name == 'pull_request' | |
| run: turbo run bench:unit --filter "${{ matrix.name }}" | |
| - name: Checkout head branch | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| clean: false | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Install tools | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prune repository | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.name }} | |
| @rust/hash-repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Run head benches | |
| run: turbo run bench:unit --filter "${{ matrix.name }}" | |
| - name: Analyze benchmarks | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.name }}" | sed 's|@||g' | sed 's|/|.|g') | |
| echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV | |
| echo '<details><summary><h2>${{ matrix.name }} – Units</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph | |
| cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| - name: Authenticate Vault | |
| id: secrets | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 | |
| with: | |
| exportToken: true | |
| url: ${{ env.VAULT_ADDR }} | |
| method: jwt | |
| role: dev | |
| secrets: | | |
| aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID; | |
| aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY; | |
| aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN; | |
| - name: Upload benchmark results | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph | |
| - name: Upload benchmark summary | |
| if: github.event_name == 'pull_request' | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }} | |
| path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md | |
| integration-benches: | |
| name: Integration | |
| needs: [setup] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| matrix: ${{ fromJSON(needs.setup.outputs.integration) }} | |
| fail-fast: false | |
| env: | |
| HASH_GRAPH_PG_DATABASE: graph | |
| if: needs.setup.outputs.integration != '{"name":[],"include":[]}' | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout base branch | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| - name: Install tools | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prune repository | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.name }} | |
| @rust/hash-repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Create temp files and folders | |
| run: mkdir -p var/logs | |
| - name: Cache Rust dependencies | |
| if: github.event_name == 'pull_request' | |
| uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 | |
| with: | |
| workspaces: ${{ matrix.path }} | |
| save-if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }} | |
| - name: Launch external services | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| touch .env.local | |
| echo 'HASH_GRAPH_PG_DATABASE=graph' > .env.local | |
| cp .env.local .env.test.local | |
| yarn external-services:test up --wait | |
| - name: Start background tasks | |
| if: github.event_name == 'pull_request' | |
| id: base-background-tasks | |
| env: | |
| # Set terminal logging to WARN level and enable verbose file logging for CI | |
| HASH_GRAPH_LOG_CONSOLE_LEVEL: warn | |
| HASH_GRAPH_LOG_FILE_ENABLED: true | |
| HASH_GRAPH_LOG_FILE_LEVEL: info | |
| HASH_GRAPH_LOG_FOLDER: ${{ github.workspace }}/var/logs | |
| run: | | |
| # Optimistically compile background tasks. | |
| turbo run compile:release --env-mode=loose | |
| mkdir -p logs | |
| turbo run start --env-mode=loose --log-order stream >var/logs/base-background-task.log 2>&1 & | |
| PID=$! | |
| echo "pid=$PID" | tee -a $GITHUB_OUTPUT | |
| turbo run start:healthcheck --env-mode=loose | |
| - name: Run base benches | |
| if: github.event_name == 'pull_request' | |
| run: turbo run bench:integration --filter "${{ matrix.name }}" | |
| - name: Show background tasks logs | |
| if: always() && github.event_name == 'pull_request' | |
| run: | | |
| kill -15 ${{ steps.base-background-tasks.outputs.pid }} || true | |
| cat var/logs/base-background-task.log | |
| - name: Tear down external services | |
| if: github.event_name == 'pull_request' | |
| run: yarn workspace @apps/hash-external-services deploy:test down | |
| - name: Checkout head branch | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| clean: false | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Install tools | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prune repository | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.name }} | |
| @rust/hash-repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Create temp files and folders | |
| run: mkdir -p var/logs | |
| - name: Launch external services | |
| run: | | |
| touch .env.local | |
| echo 'HASH_GRAPH_PG_DATABASE=graph' > .env.local | |
| cp .env.local .env.test.local | |
| yarn external-services:test up --wait | |
| - name: Start background tasks | |
| id: head-background-tasks | |
| env: | |
| # Set terminal logging to WARN level and enable verbose file logging for CI | |
| HASH_GRAPH_LOG_CONSOLE_LEVEL: warn | |
| HASH_GRAPH_LOG_FILE_ENABLED: true | |
| HASH_GRAPH_LOG_FILE_LEVEL: info | |
| HASH_GRAPH_LOG_FOLDER: ${{ github.workspace }}/var/logs | |
| run: | | |
| # Optimistically compile background tasks. | |
| turbo run compile:release --env-mode=loose | |
| mkdir -p logs | |
| turbo run start --env-mode=loose --log-order stream >var/logs/head-background-task.log 2>&1 & | |
| PID=$! | |
| echo "pid=$PID" | tee -a $GITHUB_OUTPUT | |
| turbo run start:healthcheck --env-mode=loose | |
| - name: Run head benches | |
| run: turbo run bench:integration --filter "${{ matrix.name }}" | |
| - name: Show background tasks logs | |
| if: always() | |
| run: | | |
| kill -15 ${{ steps.head-background-tasks.outputs.pid }} || true | |
| cat var/logs/head-background-task.log | |
| - name: Analyze benchmarks | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.name }}" | sed 's|@||g' | sed 's|/|.|g') | |
| echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV | |
| echo '<details><summary><h2>${{ matrix.name }} – Integrations</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph | |
| cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| - name: Authenticate Vault | |
| id: secrets | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 | |
| with: | |
| exportToken: true | |
| url: ${{ env.VAULT_ADDR }} | |
| method: jwt | |
| role: dev | |
| secrets: | | |
| aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID; | |
| aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY; | |
| aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN; | |
| - name: Upload benchmark results | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph | |
| - name: Upload benchmark summary | |
| if: github.event_name == 'pull_request' | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }} | |
| path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md | |
| - name: Show container logs | |
| if: ${{ success() || failure() }} | |
| run: yarn workspace @apps/hash-external-services deploy logs --timestamps | |
| - name: Upload background tasks logs | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| if: success() || failure() | |
| with: | |
| name: ${{ env.TRIMMED_PACKAGE_NAME }}_logs | |
| path: | | |
| var/logs | |
| passed: | |
| name: Benches passed | |
| needs: [setup, unit-benches, integration-benches] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Check setup script | |
| run: | | |
| [[ ${{ needs.setup.result }} = success ]] | |
| - name: Check unit benches | |
| run: | | |
| [[ ${{ needs.unit-benches.result }} =~ success|skipped ]] | |
| - name: Check integration benches | |
| run: | | |
| [[ ${{ needs.integration-benches.result }} =~ success|skipped ]] | |
| - name: Notify Slack on failure | |
| uses: rtCamp/action-slack-notify@07cbdbfd6c6190970778d8f98f11d073b2932aae | |
| if: ${{ failure() && github.event_name == 'merge_group' }} | |
| env: | |
| SLACK_LINK_NAMES: true | |
| SLACK_MESSAGE: "At least one bench job failed for a Pull Request in the Merge Queue failed <@U02NLJY0FGX>" # Notifies T | |
| SLACK_TITLE: Tests failed | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_USERNAME: GitHub | |
| VAULT_ADDR: "" | |
| VAULT_TOKEN: "" | |
| - name: Checkout | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Download reports' artifacts | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 | |
| with: | |
| path: downloads | |
| - name: Create comment | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| run: | | |
| echo "## Benchmark results" > /tmp/summary.md | |
| cat downloads/*/*.md >> /tmp/summary.md | |
| cat /tmp/summary.md | |
| - name: Post comment | |
| uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' && github.event.pull_request.head.repo.full_name == github.repository | |
| with: | |
| message-path: /tmp/summary.md | |
| message-id: benchmark-results | |
| refresh-message-position: true |