revert(ci): drop ineffective PYTEST_ADDOPTS xprocess workaround #154
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scenario Matrix (develop) | |
| # Runs the full scenario matrix against real sandboxed external services. | |
| # - Disabled by default on `develop` pushes; set repository variable | |
| # `ELIZA_SCENARIO_MATRIX_ENABLED=true` or use workflow_dispatch with | |
| # `enabled=true` to run it. | |
| # - NOT run on PRs — PRs run a smaller scenario-critical-subset job. | |
| # - All credentials are ELIZA_E2E_* GitHub Actions secrets sourced from the | |
| # `eliza-e2e` 1Password vault. | |
| # | |
| # 1Password vault: the `eliza-e2e` vault must exist in 1Password and the | |
| # corresponding `ELIZA_E2E_*` GitHub Actions secrets must be configured on | |
| # the `elizaOS/eliza` repo before this workflow can run successfully. | |
| # | |
| # Sharding: 8 shards total. Apple-ecosystem shards (iMessage, BlueBubbles) run on a | |
| # self-hosted macOS runner tagged `eliza-e2e-macos`; everything else runs on | |
| # GitHub-hosted `ubuntu-24.04`. | |
| # TODO: the `eliza-e2e-macos` self-hosted runner must be registered on the | |
| # `elizaOS/eliza` repo settings (Settings → Actions → Runners) before any | |
| # Apple-ecosystem shard can execute. | |
| # | |
| # This workflow remains opt-in until the scenario catalog is stable enough to be | |
| # a develop gate. | |
| on: | |
| push: | |
| branches: [develop] | |
| workflow_dispatch: | |
| inputs: | |
| enabled: | |
| description: "Enable the scenario matrix run (false = no-op)" | |
| required: false | |
| default: "false" | |
| shards: | |
| description: "Comma-separated shard names to run (empty = all)" | |
| required: false | |
| default: "" | |
| concurrency: | |
| group: scenario-matrix-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CI: "true" | |
| BUN_VERSION: "1.3.13" | |
| NODE_VERSION: "22.20.0" | |
| NODE_OPTIONS: "--max-old-space-size=8192" | |
| ELIZA_LIVE_TEST: "1" | |
| ELIZA_LIVE_SCENARIO_TEST: "1" | |
| jobs: | |
| disabled: | |
| name: Scenario Matrix Disabled | |
| if: ${{ inputs.enabled != 'true' && vars.ELIZA_SCENARIO_MATRIX_ENABLED != 'true' }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Explain disabled state | |
| run: | | |
| echo "Scenario Matrix is disabled for this run." | |
| echo "Set repository variable ELIZA_SCENARIO_MATRIX_ENABLED=true or dispatch with enabled=true to run it." | |
| matrix: | |
| name: "scenario-matrix: ${{ matrix.shard.name }}" | |
| if: ${{ inputs.enabled == 'true' || vars.ELIZA_SCENARIO_MATRIX_ENABLED == 'true' }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 8 | |
| matrix: | |
| shard: | |
| - name: messaging | |
| runs-on: ubuntu-24.04 | |
| globs: "eliza/test/scenarios/messaging.*/**/*.scenario.ts" | |
| - name: messaging-apple | |
| runs-on: [self-hosted, eliza-e2e-macos] | |
| globs: "eliza/test/scenarios/messaging.imessage/**/*.scenario.ts eliza/test/scenarios/messaging.signal/**/*.scenario.ts" | |
| - name: todos-reminders | |
| runs-on: ubuntu-24.04 | |
| globs: "eliza/test/scenarios/todos/**/*.scenario.ts eliza/test/scenarios/reminders/**/*.scenario.ts" | |
| - name: calendar | |
| runs-on: ubuntu-24.04 | |
| globs: "eliza/test/scenarios/calendar/**/*.scenario.ts" | |
| - name: relationships | |
| runs-on: ubuntu-24.04 | |
| globs: "eliza/test/scenarios/relationships/**/*.scenario.ts" | |
| - name: selfcontrol-activity | |
| runs-on: [self-hosted, eliza-e2e-macos] | |
| globs: "eliza/test/scenarios/selfcontrol/**/*.scenario.ts eliza/test/scenarios/activity/**/*.scenario.ts" | |
| - name: browser-social | |
| runs-on: ubuntu-24.04 | |
| globs: "eliza/test/scenarios/browser.*/**/*.scenario.ts eliza/test/scenarios/social.*/**/*.scenario.ts" | |
| - name: cross-cutting-gateway | |
| runs-on: ubuntu-24.04 | |
| globs: "eliza/test/scenarios/cross-cutting/**/*.scenario.ts eliza/test/scenarios/gateway/**/*.scenario.ts eliza/test/scenarios/remote/**/*.scenario.ts eliza/test/scenarios/lifeops.*/**/*.scenario.ts eliza/test/scenarios/goals/**/*.scenario.ts eliza/test/scenarios/convo/**/*.scenario.ts" | |
| - name: lifeops-app | |
| runs-on: ubuntu-24.04 | |
| root: apps/app-lifeops/test/scenarios | |
| globs: "**/*.scenario.ts" | |
| runs-on: ${{ matrix.shard.runs-on }} | |
| timeout-minutes: 60 | |
| env: | |
| # LLM providers under upstream `ELIZA_E2E_*` alias names that the | |
| # scenario-runner's live-provider helper checks. Falls back to plain | |
| # repo-level provider keys when the eliza-e2e vault entries are absent. | |
| ELIZA_E2E_GROQ_API_KEY: ${{ secrets.ELIZA_E2E_GROQ_API_KEY || secrets.GROQ_API_KEY }} | |
| ELIZA_E2E_OPENAI_API_KEY: ${{ secrets.ELIZA_E2E_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} | |
| ELIZA_E2E_ANTHROPIC_API_KEY: ${{ secrets.ELIZA_E2E_ANTHROPIC_API_KEY || secrets.ANTHROPIC_API_KEY }} | |
| ELIZA_E2E_GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.ELIZA_E2E_GOOGLE_GENERATIVE_AI_API_KEY || secrets.GOOGLE_GENERATIVE_AI_API_KEY || secrets.GOOGLE_API_KEY }} | |
| ELIZA_E2E_OPENROUTER_API_KEY: ${{ secrets.ELIZA_E2E_OPENROUTER_API_KEY || secrets.OPENROUTER_API_KEY }} | |
| # Gmail / Google Workspace | |
| ELIZA_E2E_GMAIL_TESTOWNER_CLIENT_ID: ${{ secrets.ELIZA_E2E_GMAIL_TESTOWNER_CLIENT_ID }} | |
| ELIZA_E2E_GMAIL_TESTOWNER_CLIENT_SECRET: ${{ secrets.ELIZA_E2E_GMAIL_TESTOWNER_CLIENT_SECRET }} | |
| ELIZA_E2E_GMAIL_TESTOWNER_REFRESH_TOKEN: ${{ secrets.ELIZA_E2E_GMAIL_TESTOWNER_REFRESH_TOKEN }} | |
| ELIZA_E2E_GMAIL_TESTOWNER_ADDRESS: ${{ secrets.ELIZA_E2E_GMAIL_TESTOWNER_ADDRESS }} | |
| ELIZA_E2E_GMAIL_TESTAGENT_CLIENT_ID: ${{ secrets.ELIZA_E2E_GMAIL_TESTAGENT_CLIENT_ID }} | |
| ELIZA_E2E_GMAIL_TESTAGENT_CLIENT_SECRET: ${{ secrets.ELIZA_E2E_GMAIL_TESTAGENT_CLIENT_SECRET }} | |
| ELIZA_E2E_GMAIL_TESTAGENT_REFRESH_TOKEN: ${{ secrets.ELIZA_E2E_GMAIL_TESTAGENT_REFRESH_TOKEN }} | |
| ELIZA_E2E_GMAIL_TESTAGENT_ADDRESS: ${{ secrets.ELIZA_E2E_GMAIL_TESTAGENT_ADDRESS }} | |
| # Calendly | |
| ELIZA_E2E_CALENDLY_ACCESS_TOKEN: ${{ secrets.ELIZA_E2E_CALENDLY_ACCESS_TOKEN }} | |
| ELIZA_E2E_CALENDLY_HOST_URI: ${{ secrets.ELIZA_E2E_CALENDLY_HOST_URI }} | |
| ELIZA_E2E_CALENDLY_EVENT_TYPE_URI: ${{ secrets.ELIZA_E2E_CALENDLY_EVENT_TYPE_URI }} | |
| # Discord | |
| ELIZA_E2E_DISCORD_BOT_TOKEN: ${{ secrets.ELIZA_E2E_DISCORD_BOT_TOKEN }} | |
| ELIZA_E2E_DISCORD_CLIENT_ID: ${{ secrets.ELIZA_E2E_DISCORD_CLIENT_ID }} | |
| ELIZA_E2E_DISCORD_CLIENT_SECRET: ${{ secrets.ELIZA_E2E_DISCORD_CLIENT_SECRET }} | |
| ELIZA_E2E_DISCORD_QA_GUILD_ID: ${{ secrets.ELIZA_E2E_DISCORD_QA_GUILD_ID }} | |
| ELIZA_E2E_DISCORD_QA_CHANNEL_ID: ${{ secrets.ELIZA_E2E_DISCORD_QA_CHANNEL_ID }} | |
| ELIZA_E2E_DISCORD_USER_RELAY_TOKEN: ${{ secrets.ELIZA_E2E_DISCORD_USER_RELAY_TOKEN }} | |
| # Telegram | |
| ELIZA_E2E_TELEGRAM_BOT_TOKEN: ${{ secrets.ELIZA_E2E_TELEGRAM_BOT_TOKEN }} | |
| ELIZA_E2E_TELEGRAM_APP_ID: ${{ secrets.ELIZA_E2E_TELEGRAM_APP_ID }} | |
| ELIZA_E2E_TELEGRAM_APP_HASH: ${{ secrets.ELIZA_E2E_TELEGRAM_APP_HASH }} | |
| ELIZA_E2E_TELEGRAM_USERBOT_PHONE_NUMBER: ${{ secrets.ELIZA_E2E_TELEGRAM_USERBOT_PHONE_NUMBER }} | |
| ELIZA_E2E_TELEGRAM_USERBOT_SESSION_STRING: ${{ secrets.ELIZA_E2E_TELEGRAM_USERBOT_SESSION_STRING }} | |
| ELIZA_E2E_TELEGRAM_CHAT_ID: ${{ secrets.ELIZA_E2E_TELEGRAM_CHAT_ID }} | |
| # Twitter / X | |
| ELIZA_E2E_TWITTER_CLIENT_ID: ${{ secrets.ELIZA_E2E_TWITTER_CLIENT_ID }} | |
| ELIZA_E2E_TWITTER_CLIENT_SECRET: ${{ secrets.ELIZA_E2E_TWITTER_CLIENT_SECRET }} | |
| ELIZA_E2E_TWITTER_USER_REFRESH_TOKEN: ${{ secrets.ELIZA_E2E_TWITTER_USER_REFRESH_TOKEN }} | |
| ELIZA_E2E_TWITTER_FRIEND_REFRESH_TOKEN: ${{ secrets.ELIZA_E2E_TWITTER_FRIEND_REFRESH_TOKEN }} | |
| ELIZA_E2E_TWITTER_USER_HANDLE: ${{ secrets.ELIZA_E2E_TWITTER_USER_HANDLE }} | |
| ELIZA_E2E_TWITTER_FRIEND_HANDLE: ${{ secrets.ELIZA_E2E_TWITTER_FRIEND_HANDLE }} | |
| # Signal | |
| ELIZA_E2E_SIGNAL_PHONE_NUMBER: ${{ secrets.ELIZA_E2E_SIGNAL_PHONE_NUMBER }} | |
| ELIZA_E2E_SIGNAL_RECIPIENT_PHONE_NUMBER: ${{ secrets.ELIZA_E2E_SIGNAL_RECIPIENT_PHONE_NUMBER }} | |
| ELIZA_E2E_SIGNAL_DATA_DIR: ${{ secrets.ELIZA_E2E_SIGNAL_DATA_DIR }} | |
| ELIZA_E2E_WHATSAPP_ACCESS_TOKEN: ${{ secrets.ELIZA_E2E_WHATSAPP_ACCESS_TOKEN }} | |
| ELIZA_E2E_WHATSAPP_PHONE_NUMBER_ID: ${{ secrets.ELIZA_E2E_WHATSAPP_PHONE_NUMBER_ID }} | |
| ELIZA_E2E_WHATSAPP_BUSINESS_ACCOUNT_ID: ${{ secrets.ELIZA_E2E_WHATSAPP_BUSINESS_ACCOUNT_ID }} | |
| ELIZA_E2E_WHATSAPP_WEBHOOK_VERIFY_TOKEN: ${{ secrets.ELIZA_E2E_WHATSAPP_WEBHOOK_VERIFY_TOKEN }} | |
| ELIZA_E2E_WHATSAPP_RECIPIENT_PHONE_NUMBER: ${{ secrets.ELIZA_E2E_WHATSAPP_RECIPIENT_PHONE_NUMBER }} | |
| # Twilio | |
| ELIZA_E2E_TWILIO_ACCOUNT_SID: ${{ secrets.ELIZA_E2E_TWILIO_ACCOUNT_SID }} | |
| ELIZA_E2E_TWILIO_API_KEY_SID: ${{ secrets.ELIZA_E2E_TWILIO_API_KEY_SID }} | |
| ELIZA_E2E_TWILIO_API_KEY_SECRET: ${{ secrets.ELIZA_E2E_TWILIO_API_KEY_SECRET }} | |
| ELIZA_E2E_TWILIO_SMS_FROM: ${{ secrets.ELIZA_E2E_TWILIO_SMS_FROM }} | |
| ELIZA_E2E_TWILIO_VOICE_FROM: ${{ secrets.ELIZA_E2E_TWILIO_VOICE_FROM }} | |
| ELIZA_E2E_TWILIO_MESSAGING_SERVICE_SID: ${{ secrets.ELIZA_E2E_TWILIO_MESSAGING_SERVICE_SID }} | |
| ELIZA_E2E_TWILIO_RECIPIENT: ${{ secrets.ELIZA_E2E_TWILIO_RECIPIENT }} | |
| # BlueBubbles | |
| ELIZA_E2E_BLUEBUBBLES_SERVER_URL: ${{ secrets.ELIZA_E2E_BLUEBUBBLES_SERVER_URL }} | |
| ELIZA_E2E_BLUEBUBBLES_PASSWORD: ${{ secrets.ELIZA_E2E_BLUEBUBBLES_PASSWORD }} | |
| ELIZA_E2E_BLUEBUBBLES_RECIPIENT_HANDLE: ${{ secrets.ELIZA_E2E_BLUEBUBBLES_RECIPIENT_HANDLE }} | |
| # GitHub | |
| ELIZA_E2E_GITHUB_USER_PAT: ${{ secrets.ELIZA_E2E_GITHUB_USER_PAT }} | |
| ELIZA_E2E_GITHUB_AGENT_PAT: ${{ secrets.ELIZA_E2E_GITHUB_AGENT_PAT }} | |
| ELIZA_E2E_GITHUB_ORG: ${{ secrets.ELIZA_E2E_GITHUB_ORG }} | |
| ELIZA_E2E_GITHUB_TEMPLATE_REPO: ${{ secrets.ELIZA_E2E_GITHUB_TEMPLATE_REPO }} | |
| # 1Password autofill vault | |
| ELIZA_E2E_ONEPASS_SA_TOKEN: ${{ secrets.ELIZA_E2E_ONEPASS_SA_TOKEN }} | |
| ELIZA_E2E_ONEPASS_VAULT_ID: ${{ secrets.ELIZA_E2E_ONEPASS_VAULT_ID }} | |
| # Eliza Cloud | |
| ELIZA_E2E_ELIZACLOUD_API_KEY: ${{ secrets.ELIZA_E2E_ELIZACLOUD_API_KEY }} | |
| ELIZA_E2E_ELIZACLOUD_BASE_URL: ${{ secrets.ELIZA_E2E_ELIZACLOUD_BASE_URL }} | |
| # Apple | |
| ELIZA_E2E_APPLE_TEAM_ID: ${{ secrets.ELIZA_E2E_APPLE_TEAM_ID }} | |
| ELIZA_E2E_APPLE_APNS_KEY_ID: ${{ secrets.ELIZA_E2E_APPLE_APNS_KEY_ID }} | |
| ELIZA_E2E_APPLE_APNS_KEY_P8: ${{ secrets.ELIZA_E2E_APPLE_APNS_KEY_P8 }} | |
| ELIZA_E2E_APPLE_APNS_TOPIC: ${{ secrets.ELIZA_E2E_APPLE_APNS_TOPIC }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| submodules: false | |
| fetch-depth: 0 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: ${{ env.BUN_VERSION }} | |
| - name: Setup workspace dependencies | |
| uses: ./.github/actions/setup-bun-workspace | |
| with: | |
| bun-version: ${{ env.BUN_VERSION }} | |
| install-command: bun install --ignore-scripts --no-frozen-lockfile | |
| install-native-deps: "false" | |
| - name: Run scenario shard | |
| id: run | |
| # TODO: scenario-runner CLI bin is still named `milady-scenarios` in | |
| # packages/scenario-runner/package.json. Rename to `eliza-scenarios` | |
| # in a follow-up and update the command below. | |
| run: | | |
| set -f | |
| bun run milady-scenarios run \ | |
| ${{ matrix.shard.root || 'eliza/test/scenarios' }} \ | |
| --report-dir reports/scenarios/${{ github.run_id }}-${{ matrix.shard.name }} \ | |
| --runId "${{ github.run_id }}-${{ matrix.shard.name }}" \ | |
| ${{ matrix.shard.globs }} | |
| continue-on-error: false | |
| - name: Upload shard report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: scenario-report-${{ matrix.shard.name }} | |
| path: reports/scenarios/${{ github.run_id }}-${{ matrix.shard.name }}/ | |
| summary: | |
| name: Matrix summary | |
| needs: [matrix] | |
| if: ${{ always() && needs.matrix.result != 'skipped' }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| submodules: false | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: ${{ env.BUN_VERSION }} | |
| - name: Download all shard reports | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: scenario-report-* | |
| path: reports/scenarios/all | |
| merge-multiple: true | |
| - name: Aggregate | |
| run: bun run scripts/aggregate-scenario-reports.mjs reports/scenarios/all >> "$GITHUB_STEP_SUMMARY" |