E2E Tests #25276
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| workflow_run: | |
| workflows: ['Quality assurance'] | |
| types: | |
| - completed | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.workflow_run.head_sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-changes: | |
| if: ${{ github.repository == 'mastra-ai/mastra' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| statuses: write | |
| contents: read | |
| outputs: | |
| e2e-changed: ${{ steps.changes.outputs.e2e }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - name: Set pending status | |
| uses: ./.github/workflows/shared-actions/set-pr-status | |
| with: | |
| status: 'pending' | |
| context: 'E2E Tests' | |
| description: 'Checking for changes' | |
| sha: ${{ github.event.workflow_run.head_sha }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - name: Check for changes that require e2e tests | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: main | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| filters: | | |
| e2e: | |
| - '!(**/*.md)packages/**' | |
| - '!(**/*.md)e2e-tests/**' | |
| - '!(**/*.md)stores/**' | |
| - '!(**/*.md)client-sdks/**' | |
| - '!(**/*.md)deployers/**' | |
| - '.github/workflows/secrets.e2e.yml' | |
| skip-tests: | |
| needs: check-changes | |
| if: needs.check-changes.outputs.e2e-changed == 'false' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| statuses: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set success status for unchanged e2e | |
| uses: ./.github/workflows/shared-actions/set-pr-status | |
| with: | |
| status: 'success' | |
| context: 'E2E Tests' | |
| description: 'No changes requiring e2e tests - skipping' | |
| sha: ${{ github.event.workflow_run.head_sha }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| e2e-monorepo: | |
| name: E2E monorepo | |
| needs: check-changes | |
| if: ${{ github.repository == 'mastra-ai/mastra' && needs.check-changes.outputs.e2e-changed == 'true' }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| # os: [ubuntu-latest, windows-latest] # <- windows is skipped for now because the CI setup doesn't work and is blocking PR merges | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: ${{ secrets.TURBO_TEAM }} | |
| TURBO_CACHE: remote:r | |
| GITHUB_REF_NAME: ${{ github.event.workflow_run.head_branch }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| run_install: false | |
| - name: Setup Node.js 22.x | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22.13.0 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Build | |
| run: pnpm turbo --filter "./packages/core" --filter "./packages/cli" build | |
| - name: Install e2e test dependencies | |
| working-directory: ./e2e-tests/monorepo | |
| run: pnpm install --ignore-workspace | |
| - name: Test | |
| working-directory: ./e2e-tests/monorepo | |
| run: pnpm test | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| e2e-no-bundling: | |
| name: E2E no-bundling | |
| needs: check-changes | |
| if: ${{ github.repository == 'mastra-ai/mastra' && needs.check-changes.outputs.e2e-changed == 'true' }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| # os: [ubuntu-latest, windows-latest] # <- windows is skipped for now because the CI setup doesn't work and is blocking PR merges | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: ${{ secrets.TURBO_TEAM }} | |
| TURBO_CACHE: remote:r | |
| GITHUB_REF_NAME: ${{ github.event.workflow_run.head_branch }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| run_install: false | |
| - name: Setup Node.js 22.x | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22.13.0 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Build | |
| run: pnpm turbo --filter "@mastra/core" --filter "@mastra/evals" --filter "@mastra/libsql" --filter "@mastra/loggers" --filter "@mastra/memory" --filter "@mastra/observability" --filter "mastra" build | |
| - name: Install e2e test dependencies | |
| working-directory: ./e2e-tests/no-bundling | |
| run: pnpm install --ignore-workspace | |
| - name: Test | |
| working-directory: ./e2e-tests/no-bundling | |
| run: pnpm test | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| e2e-create-mastra: | |
| name: E2E create-mastra | |
| needs: check-changes | |
| if: ${{ github.repository == 'mastra-ai/mastra' && needs.check-changes.outputs.e2e-changed == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: ${{ secrets.TURBO_TEAM }} | |
| TURBO_CACHE: remote:r | |
| GITHUB_REF_NAME: ${{ github.event.workflow_run.head_branch }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| run_install: false | |
| - name: Setup Node.js 22.x | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22.13.0 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Build | |
| run: pnpm turbo --filter "create-mastra" build | |
| - name: Install e2e test dependencies | |
| working-directory: ./e2e-tests/create-mastra | |
| run: pnpm install --ignore-workspace | |
| - name: Test | |
| working-directory: ./e2e-tests/create-mastra | |
| run: pnpm test | |
| e2e-commonjs: | |
| name: E2E CommonJS | |
| needs: check-changes | |
| if: ${{ github.repository == 'mastra-ai/mastra' && needs.check-changes.outputs.e2e-changed == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: ${{ secrets.TURBO_TEAM }} | |
| TURBO_CACHE: remote:r | |
| GITHUB_REF_NAME: ${{ github.event.workflow_run.head_branch }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| run_install: false | |
| - name: Setup Node.js 22.x | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22.13.0 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Build | |
| run: pnpm turbo --filter "@mastra/core" --filter "@mastra/loggers" build | |
| - name: Install e2e test dependencies | |
| working-directory: ./e2e-tests/commonjs | |
| run: pnpm install --ignore-workspace | |
| - name: Test | |
| working-directory: ./e2e-tests/commonjs | |
| run: pnpm test | |
| e2e-deployers: | |
| name: E2E deployers | |
| needs: check-changes | |
| if: ${{ github.repository == 'mastra-ai/mastra' && needs.check-changes.outputs.e2e-changed == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| permissions: | |
| contents: read | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: ${{ secrets.TURBO_TEAM }} | |
| TURBO_CACHE: remote:r | |
| GITHUB_REF_NAME: ${{ github.event.workflow_run.head_branch }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| run_install: false | |
| - name: Setup Node.js 22.x | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22.13.0 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Build | |
| run: pnpm build | |
| - name: Install e2e test dependencies | |
| working-directory: ./e2e-tests/deployers | |
| run: pnpm install --ignore-workspace | |
| - name: Test | |
| working-directory: ./e2e-tests/deployers | |
| run: pnpm test | |
| e2e-type-check: | |
| name: E2E Type check | |
| needs: check-changes | |
| if: ${{ github.repository == 'mastra-ai/mastra' && needs.check-changes.outputs.e2e-changed == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| permissions: | |
| contents: read | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: ${{ secrets.TURBO_TEAM }} | |
| TURBO_CACHE: remote:r | |
| GITHUB_REF_NAME: ${{ github.event.workflow_run.head_branch }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| run_install: false | |
| - name: Setup Node.js 22.x | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22.13.0 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Build | |
| run: pnpm build | |
| - name: Install e2e test type-check | |
| working-directory: ./e2e-tests/type-check | |
| run: pnpm install --ignore-workspace | |
| - name: Test | |
| working-directory: ./e2e-tests/type-check | |
| run: pnpm test | |
| e2e-kitchen-sink: | |
| name: E2E kitchen-sink | |
| needs: check-changes | |
| if: ${{ github.repository == 'mastra-ai/mastra' && needs.check-changes.outputs.e2e-changed == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: ${{ secrets.TURBO_TEAM }} | |
| TURBO_CACHE: remote:r | |
| GITHUB_REF_NAME: ${{ github.event.workflow_run.head_branch }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| run_install: false | |
| - name: Setup Node.js 22.x | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22.13.0 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Build | |
| run: pnpm build | |
| - name: Setup e2e project dependencies | |
| working-directory: ./packages/playground | |
| run: pnpm test:e2e:setup | |
| - name: Test E2E | |
| working-directory: ./packages/playground | |
| run: pnpm test:e2e | |
| e2e-client-js: | |
| name: E2E client-js (optional) | |
| needs: check-changes | |
| if: ${{ github.repository == 'mastra-ai/mastra' && needs.check-changes.outputs.e2e-changed == 'true' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| continue-on-error: true | |
| permissions: | |
| contents: read | |
| env: | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: ${{ secrets.TURBO_TEAM }} | |
| TURBO_CACHE: remote:r | |
| GITHUB_REF_NAME: ${{ github.event.workflow_run.head_branch }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.workflow_run.head_sha }} | |
| - uses: pnpm/action-setup@v4 | |
| name: Install pnpm | |
| with: | |
| run_install: false | |
| - name: Setup Node.js 22.x | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22.13.0 | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: Build | |
| run: pnpm build | |
| - name: Test client-js zod-v3 | |
| run: pnpm --filter "client-js-e2e-tests-zod-v3" test | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| - name: Test client-js zod-v4 | |
| run: pnpm --filter "client-js-e2e-tests-zod-v4" test | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| e2e-success: | |
| needs: | |
| [ | |
| check-changes, | |
| e2e-monorepo, | |
| e2e-create-mastra, | |
| e2e-commonjs, | |
| e2e-deployers, | |
| e2e-kitchen-sink, | |
| e2e-no-bundling, | |
| e2e-type-check, | |
| ] | |
| if: ${{ always() && needs.check-changes.outputs.e2e-changed == 'true' && needs.e2e-monorepo.result == 'success' && needs.e2e-create-mastra.result == 'success' && needs.e2e-commonjs.result == 'success' && needs.e2e-deployers.result == 'success' && needs.e2e-kitchen-sink.result == 'success' && needs.e2e-no-bundling.result == 'success' && needs.e2e-type-check.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| statuses: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set success status for completed tests | |
| uses: ./.github/workflows/shared-actions/set-pr-status | |
| with: | |
| status: 'success' | |
| context: 'E2E Tests' | |
| description: 'All E2E tests passed' | |
| sha: ${{ github.event.workflow_run.head_sha }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| e2e-failure: | |
| needs: | |
| [ | |
| check-changes, | |
| e2e-monorepo, | |
| e2e-create-mastra, | |
| e2e-commonjs, | |
| e2e-deployers, | |
| e2e-kitchen-sink, | |
| e2e-no-bundling, | |
| e2e-type-check, | |
| ] | |
| if: ${{ always() && needs.check-changes.outputs.e2e-changed == 'true' && (needs.e2e-monorepo.result == 'failure' || needs.e2e-create-mastra.result == 'failure' || needs.e2e-commonjs.result == 'failure' || needs.e2e-deployers.result == 'failure' || needs.e2e-kitchen-sink.result == 'failure' || needs.e2e-no-bundling.result == 'failure' || needs.e2e-type-check.result == 'failure') }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| statuses: write | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Set failure status for failed tests | |
| uses: ./.github/workflows/shared-actions/set-pr-status | |
| with: | |
| status: 'failure' | |
| context: 'E2E Tests' | |
| description: 'One or more E2E tests failed' | |
| sha: ${{ github.event.workflow_run.head_sha }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} |