Update LLM provider SDK npm packages #12950
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bench | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: hashintel | |
| TURBO_CACHE: remote:rw | |
| VAULT_ADDR: ${{ secrets.VAULT_ADDR }} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| optimize-ci: # Do not run benches in the middle of the stack | |
| runs-on: ubuntu-24.04 | |
| name: Optimize CI | |
| outputs: | |
| skip: ${{ steps.check_skip.outputs.skip }} | |
| steps: | |
| - name: Optimize CI | |
| id: check_skip | |
| uses: withgraphite/graphite-ci-action@6213dcfab02da542aaf534e087282a928ff4374a | |
| with: | |
| graphite_token: ${{ secrets.GRAPHITE_CI_OPTIMIZER_TOKEN }} | |
| setup: | |
| needs: [optimize-ci] | |
| if: needs.optimize-ci.outputs.skip == 'false' | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| unit: ${{ steps.packages.outputs.unit }} | |
| integration: ${{ steps.packages.outputs.integration }} | |
| steps: | |
| - name: Checkout source code | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install tools | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| rust: false | |
| - name: Determine changed packages | |
| id: packages | |
| run: | | |
| UNIT_BENCH_QUERY='query { affectedPackages(base: "HEAD^", filter: {has: {field: TASK_NAME, value: "bench:unit"}}) { items { name path } } }' | |
| INTEGRATION_BENCH_QUERY='query { affectedPackages(base: "HEAD^", filter: {has: {field: TASK_NAME, value: "bench:integration"}}) { items { name path } } }' | |
| UNIT_BENCH_PACKAGES=$(turbo query "$UNIT_BENCH_QUERY" \ | |
| | jq --compact-output '.data.affectedPackages.items | [(.[] | select(.name != "//"))] | { name: [.[].name], include: . }') | |
| INTEGRATION_BENCH_PACKAGES=$(turbo query "$INTEGRATION_BENCH_QUERY" \ | |
| | jq --compact-output '.data.affectedPackages.items | [(.[] | select(.name != "//"))] | { name: [.[].name], include: . }') | |
| echo "unit=$UNIT_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT | |
| echo "integration=$INTEGRATION_BENCH_PACKAGES" | tee -a $GITHUB_OUTPUT | |
| unit-benches: | |
| name: Unit | |
| needs: [setup] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| matrix: ${{ fromJSON(needs.setup.outputs.unit) }} | |
| fail-fast: false | |
| if: needs.setup.outputs.unit != '{"name":[],"include":[]}' | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout base branch | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| - name: Clean up disk | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/clean-up-disk | |
| - name: Install tools | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prune repository | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.name }} | |
| @rust/hash-repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Run base benches | |
| if: github.event_name == 'pull_request' | |
| run: turbo run bench:unit --filter "${{ matrix.name }}" | |
| - name: Checkout head branch | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| clean: false | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Clean up disk | |
| if: github.event_name != 'pull_request' | |
| uses: ./.github/actions/clean-up-disk | |
| - name: Install tools | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prune repository | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.name }} | |
| @rust/hash-repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Run head benches | |
| run: turbo run bench:unit --filter "${{ matrix.name }}" | |
| - name: Analyze benchmarks | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.name }}" | sed 's|@||g' | sed 's|/|.|g') | |
| echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV | |
| echo '<details><summary><h2>${{ matrix.name }} – Units</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph | |
| cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| - name: Authenticate Vault | |
| id: secrets | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 | |
| with: | |
| exportToken: true | |
| url: ${{ env.VAULT_ADDR }} | |
| method: jwt | |
| role: dev | |
| secrets: | | |
| aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID; | |
| aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY; | |
| aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN; | |
| - name: Upload benchmark results | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph | |
| - name: Upload benchmark summary | |
| if: github.event_name == 'pull_request' | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }} | |
| path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md | |
| integration-benches: | |
| name: Integration | |
| needs: [setup] | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| matrix: ${{ fromJSON(needs.setup.outputs.integration) }} | |
| fail-fast: false | |
| env: | |
| HASH_GRAPH_PG_DATABASE: graph | |
| if: needs.setup.outputs.integration != '{"name":[],"include":[]}' | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout base branch | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| - name: Clean up disk | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/clean-up-disk | |
| - name: Install tools | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prune repository | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.name }} | |
| @rust/hash-repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Create temp files and folders | |
| run: mkdir -p var/logs | |
| - name: Launch external services | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| touch .env.local | |
| echo 'HASH_GRAPH_PG_DATABASE=graph' > .env.local | |
| cp .env.local .env.test.local | |
| yarn external-services:test up --wait | |
| - name: Start background tasks | |
| if: github.event_name == 'pull_request' | |
| id: base-background-tasks | |
| env: | |
| # Set terminal logging to WARN level and enable verbose file logging for CI | |
| HASH_GRAPH_LOG_CONSOLE_LEVEL: warn | |
| HASH_GRAPH_LOG_FILE_ENABLED: true | |
| HASH_GRAPH_LOG_FILE_LEVEL: info | |
| HASH_GRAPH_LOG_FOLDER: ${{ github.workspace }}/var/logs | |
| run: | | |
| # Optimistically compile background tasks. | |
| turbo run compile:release --env-mode=loose | |
| mkdir -p logs | |
| turbo run start --env-mode=loose --log-order stream >var/logs/base-background-task.log 2>&1 & | |
| PID=$! | |
| echo "pid=$PID" | tee -a $GITHUB_OUTPUT | |
| turbo run start:healthcheck --env-mode=loose | |
| - name: Run base benches | |
| if: github.event_name == 'pull_request' | |
| run: turbo run bench:integration --filter "${{ matrix.name }}" | |
| - name: Show background tasks logs | |
| if: always() && github.event_name == 'pull_request' | |
| run: | | |
| kill -15 ${{ steps.base-background-tasks.outputs.pid }} || true | |
| cat var/logs/base-background-task.log | |
| - name: Tear down external services | |
| if: github.event_name == 'pull_request' | |
| run: yarn workspace @apps/hash-external-services deploy:test down | |
| - name: Checkout head branch | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| clean: false | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Checkout | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Clean up disk | |
| if: github.event_name != 'pull_request' | |
| uses: ./.github/actions/clean-up-disk | |
| - name: Install tools | |
| uses: ./.github/actions/install-tools | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prune repository | |
| uses: ./.github/actions/prune-repository | |
| with: | |
| scope: | | |
| ${{ matrix.name }} | |
| @rust/hash-repo-chores | |
| @apps/hash-external-services | |
| - name: Warm up repository | |
| uses: ./.github/actions/warm-up-repo | |
| - name: Create temp files and folders | |
| run: mkdir -p var/logs | |
| - name: Launch external services | |
| run: | | |
| touch .env.local | |
| echo 'HASH_GRAPH_PG_DATABASE=graph' > .env.local | |
| cp .env.local .env.test.local | |
| yarn external-services:test up --wait | |
| - name: Start background tasks | |
| id: head-background-tasks | |
| env: | |
| # Set terminal logging to WARN level and enable verbose file logging for CI | |
| HASH_GRAPH_LOG_CONSOLE_LEVEL: warn | |
| HASH_GRAPH_LOG_FILE_ENABLED: true | |
| HASH_GRAPH_LOG_FILE_LEVEL: info | |
| HASH_GRAPH_LOG_FOLDER: ${{ github.workspace }}/var/logs | |
| run: | | |
| # Optimistically compile background tasks. | |
| turbo run compile:release --env-mode=loose | |
| mkdir -p logs | |
| turbo run start --env-mode=loose --log-order stream >var/logs/head-background-task.log 2>&1 & | |
| PID=$! | |
| echo "pid=$PID" | tee -a $GITHUB_OUTPUT | |
| turbo run start:healthcheck --env-mode=loose | |
| - name: Run head benches | |
| run: turbo run bench:integration --filter "${{ matrix.name }}" | |
| - name: Show background tasks logs | |
| if: always() | |
| run: | | |
| kill -15 ${{ steps.head-background-tasks.outputs.pid }} || true | |
| cat var/logs/head-background-task.log | |
| - name: Analyze benchmarks | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| TRIMMED_PACKAGE_NAME=$(echo "${{ matrix.name }}" | sed 's|@||g' | sed 's|/|.|g') | |
| echo "TRIMMED_PACKAGE_NAME=$TRIMMED_PACKAGE_NAME" >> $GITHUB_ENV | |
| echo '<details><summary><h2>${{ matrix.name }} – Integrations</h2></summary>' > "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| turbo run analyze-benchmarks -- --output "/tmp/benches.md" --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph | |
| cat "/tmp/benches.md" >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| echo '</details>' >> "/tmp/$TRIMMED_PACKAGE_NAME.md" | |
| - name: Authenticate Vault | |
| id: secrets | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0 | |
| with: | |
| exportToken: true | |
| url: ${{ env.VAULT_ADDR }} | |
| method: jwt | |
| role: dev | |
| secrets: | | |
| aws/creds/dev-deploy access_key | AWS_ACCESS_KEY_ID; | |
| aws/creds/dev-deploy secret_key | AWS_SECRET_ACCESS_KEY; | |
| aws/creds/dev-deploy security_token | AWS_SESSION_TOKEN; | |
| - name: Upload benchmark results | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| run: turbo run upload-benchmarks --env-mode=loose -- --artifacts-path "$(pwd)/${{ matrix.path }}/out" --enforce-flame-graph | |
| - name: Upload benchmark summary | |
| if: github.event_name == 'pull_request' | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: ${{ env.TRIMMED_PACKAGE_NAME }}_${{ github.job }} | |
| path: /tmp/${{ env.TRIMMED_PACKAGE_NAME }}.md | |
| - name: Show container logs | |
| if: ${{ success() || failure() }} | |
| run: yarn workspace @apps/hash-external-services deploy logs --timestamps | |
| - name: Upload background tasks logs | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| if: success() || failure() | |
| with: | |
| name: ${{ env.TRIMMED_PACKAGE_NAME }}_logs | |
| path: | | |
| var/logs | |
| passed: | |
| name: Benches passed | |
| needs: [setup, unit-benches, integration-benches, optimize-ci] | |
| if: always() && needs.optimize-ci.outputs.skip == 'false' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: write | |
| steps: | |
| - name: Check setup script | |
| run: | | |
| [[ ${{ needs.setup.result }} = success ]] | |
| - name: Check unit benches | |
| run: | | |
| [[ ${{ needs.unit-benches.result }} =~ success|skipped ]] | |
| - name: Check integration benches | |
| run: | | |
| [[ ${{ needs.integration-benches.result }} =~ success|skipped ]] | |
| - name: Notify Slack on failure | |
| uses: rtCamp/action-slack-notify@07cbdbfd6c6190970778d8f98f11d073b2932aae | |
| if: ${{ failure() && github.event_name == 'merge_group' }} | |
| env: | |
| SLACK_LINK_NAMES: true | |
| SLACK_MESSAGE: "At least one bench job for a Pull Request in the Merge Queue failed <!subteam^S09KH99698T>" # Notifies @devops | |
| SLACK_TITLE: Tests failed | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_USERNAME: GitHub | |
| VAULT_ADDR: "" | |
| VAULT_TOKEN: "" | |
| - name: Checkout | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Download reports' artifacts | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 | |
| with: | |
| path: downloads | |
| - name: Create comment | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' | |
| run: | | |
| echo "## Benchmark results" > /tmp/summary.md | |
| cat downloads/*/*.md >> /tmp/summary.md | |
| cat /tmp/summary.md | |
| - name: Post comment | |
| uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2.8.2 | |
| if: github.event_name == 'pull_request' && needs.integration-benches.result == 'success' && github.event.pull_request.head.repo.full_name == github.repository | |
| with: | |
| message-path: /tmp/summary.md | |
| message-id: benchmark-results | |
| refresh-message-position: true |