E2E Flow Tests #274
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Flow Tests | |
| # This workflow runs comprehensive E2E tests with real email accounts. | |
| # It uses the pre-built Docker image from ghcr.io/elie222/inbox-zero:latest | |
| # instead of building from source, saving ~5 minutes per run. | |
| # | |
| # To enable: Set the repository variable E2E_FLOWS_ENABLED=true | |
| # To disable: Remove the variable or set it to anything other than "true" | |
| on: | |
| # Run on schedule (every 12 hours) | |
| schedule: | |
| - cron: "0 */12 * * *" | |
| # Allow manual trigger with branch selection | |
| workflow_dispatch: | |
| inputs: | |
| branch: | |
| description: "Branch to test" | |
| required: false | |
| default: "main" | |
| test_file: | |
| description: "Specific test file (optional, e.g., full-reply-cycle)" | |
| required: false | |
| default: "" | |
| # Prevent concurrent runs to avoid test account conflicts | |
| concurrency: | |
| group: e2e-flows | |
| cancel-in-progress: false | |
| env: | |
| DOCKER_IMAGE: ghcr.io/elie222/inbox-zero:latest | |
| jobs: | |
| check-enabled: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| outputs: | |
| enabled: ${{ steps.check.outputs.enabled }} | |
| steps: | |
| - name: Check if E2E flows are enabled | |
| id: check | |
| run: | | |
| if [ "${{ vars.E2E_FLOWS_ENABLED }}" = "true" ]; then | |
| echo "enabled=true" >> $GITHUB_OUTPUT | |
| echo "E2E flow tests are ENABLED" | |
| else | |
| echo "enabled=false" >> $GITHUB_OUTPUT | |
| echo "E2E flow tests are DISABLED (set E2E_FLOWS_ENABLED=true to enable)" | |
| fi | |
| e2e-flows: | |
| needs: check-enabled | |
| if: needs.check-enabled.outputs.enabled == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| ref: ${{ github.event.inputs.branch || github.ref }} | |
| - name: Setup Node.js | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: "24" | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@f40ffcd9367d9f12939873eb1018b921a783ffaa # v4 | |
| - name: Setup pnpm cache | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| with: | |
| path: ~/.pnpm-store | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pnpm-store- | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Install ngrok | |
| run: | | |
| curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null | |
| echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | sudo tee /etc/apt/sources.list.d/ngrok.list | |
| sudo apt-get update && sudo apt-get install ngrok | |
| - name: Configure ngrok | |
| run: ngrok config add-authtoken ${{ secrets.E2E_NGROK_AUTH_TOKEN }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Start app with Docker | |
| run: | | |
| docker run -d \ | |
| --pull always \ | |
| --name inbox-zero-e2e \ | |
| -p 3000:3000 \ | |
| -e DATABASE_URL="${{ secrets.DATABASE_URL }}" \ | |
| -e DIRECT_URL="${{ secrets.DIRECT_URL }}" \ | |
| -e UPSTASH_REDIS_URL="${{ secrets.UPSTASH_REDIS_URL }}" \ | |
| -e UPSTASH_REDIS_TOKEN="${{ secrets.UPSTASH_REDIS_TOKEN }}" \ | |
| -e REDIS_URL="${{ secrets.REDIS_URL }}" \ | |
| -e GOOGLE_CLIENT_ID="${{ secrets.GOOGLE_CLIENT_ID }}" \ | |
| -e GOOGLE_CLIENT_SECRET="${{ secrets.GOOGLE_CLIENT_SECRET }}" \ | |
| -e GOOGLE_PUBSUB_TOPIC_NAME="${{ secrets.GOOGLE_PUBSUB_TOPIC_NAME }}" \ | |
| -e GOOGLE_PUBSUB_VERIFICATION_TOKEN="${{ secrets.GOOGLE_PUBSUB_VERIFICATION_TOKEN }}" \ | |
| -e MICROSOFT_CLIENT_ID="${{ secrets.MICROSOFT_CLIENT_ID }}" \ | |
| -e MICROSOFT_CLIENT_SECRET="${{ secrets.MICROSOFT_CLIENT_SECRET }}" \ | |
| -e MICROSOFT_TENANT_ID="${{ secrets.MICROSOFT_TENANT_ID }}" \ | |
| -e MICROSOFT_WEBHOOK_CLIENT_STATE="${{ secrets.MICROSOFT_WEBHOOK_CLIENT_STATE }}" \ | |
| -e DEFAULT_LLM_PROVIDER="${{ secrets.DEFAULT_LLM_PROVIDER }}" \ | |
| -e DEFAULT_LLM_MODEL="${{ secrets.DEFAULT_LLM_MODEL }}" \ | |
| -e DEFAULT_OPENROUTER_PROVIDERS="${{ secrets.DEFAULT_OPENROUTER_PROVIDERS }}" \ | |
| -e ECONOMY_LLM_PROVIDER="${{ secrets.ECONOMY_LLM_PROVIDER }}" \ | |
| -e ECONOMY_LLM_MODEL="${{ secrets.ECONOMY_LLM_MODEL }}" \ | |
| -e ECONOMY_OPENROUTER_PROVIDERS="${{ secrets.ECONOMY_OPENROUTER_PROVIDERS }}" \ | |
| -e CHAT_LLM_PROVIDER="${{ secrets.CHAT_LLM_PROVIDER }}" \ | |
| -e CHAT_LLM_MODEL="${{ secrets.CHAT_LLM_MODEL }}" \ | |
| -e CHAT_OPENROUTER_PROVIDERS="${{ secrets.CHAT_OPENROUTER_PROVIDERS }}" \ | |
| -e OPENROUTER_BACKUP_MODEL="${{ secrets.OPENROUTER_BACKUP_MODEL }}" \ | |
| -e OPENAI_API_KEY="${{ secrets.OPENAI_API_KEY }}" \ | |
| -e ANTHROPIC_API_KEY="${{ secrets.ANTHROPIC_API_KEY }}" \ | |
| -e GOOGLE_API_KEY="${{ secrets.GOOGLE_API_KEY }}" \ | |
| -e OPENROUTER_API_KEY="${{ secrets.OPENROUTER_API_KEY }}" \ | |
| -e GROQ_API_KEY="${{ secrets.GROQ_API_KEY }}" \ | |
| -e PERPLEXITY_API_KEY="${{ secrets.PERPLEXITY_API_KEY }}" \ | |
| -e BEDROCK_ACCESS_KEY="${{ secrets.BEDROCK_ACCESS_KEY }}" \ | |
| -e BEDROCK_SECRET_KEY="${{ secrets.BEDROCK_SECRET_KEY }}" \ | |
| -e BEDROCK_REGION="${{ secrets.BEDROCK_REGION }}" \ | |
| -e OLLAMA_BASE_URL="${{ secrets.OLLAMA_BASE_URL }}" \ | |
| -e OLLAMA_MODEL="${{ secrets.OLLAMA_MODEL }}" \ | |
| -e AI_GATEWAY_API_KEY="${{ secrets.AI_GATEWAY_API_KEY }}" \ | |
| -e AUTH_SECRET="${{ secrets.AUTH_SECRET }}" \ | |
| -e EMAIL_ENCRYPT_SECRET="${{ secrets.EMAIL_ENCRYPT_SECRET }}" \ | |
| -e EMAIL_ENCRYPT_SALT="${{ secrets.EMAIL_ENCRYPT_SALT }}" \ | |
| -e INTERNAL_API_KEY="${{ secrets.INTERNAL_API_KEY }}" \ | |
| -e API_KEY_SALT="${{ secrets.API_KEY_SALT }}" \ | |
| -e CRON_SECRET="${{ secrets.CRON_SECRET }}" \ | |
| -e QSTASH_TOKEN="${{ secrets.QSTASH_TOKEN }}" \ | |
| -e QSTASH_CURRENT_SIGNING_KEY="${{ secrets.QSTASH_CURRENT_SIGNING_KEY }}" \ | |
| -e QSTASH_NEXT_SIGNING_KEY="${{ secrets.QSTASH_NEXT_SIGNING_KEY }}" \ | |
| -e TINYBIRD_TOKEN="${{ secrets.TINYBIRD_TOKEN }}" \ | |
| -e TINYBIRD_BASE_URL="${{ secrets.TINYBIRD_BASE_URL }}" \ | |
| -e NEXT_PUBLIC_BASE_URL="http://localhost:3000" \ | |
| -e NEXT_PUBLIC_BYPASS_PREMIUM_CHECKS="${{ secrets.NEXT_PUBLIC_BYPASS_PREMIUM_CHECKS }}" \ | |
| -e NEXT_PUBLIC_IS_RESEND_CONFIGURED="${{ secrets.NEXT_PUBLIC_IS_RESEND_CONFIGURED }}" \ | |
| -e RESEND_API_KEY="${{ secrets.RESEND_API_KEY }}" \ | |
| -e DISABLE_LOG_ZOD_ERRORS="${{ secrets.DISABLE_LOG_ZOD_ERRORS }}" \ | |
| -e LOG_TO_CONSOLE="true" \ | |
| -e ENABLE_DEBUG_LOGS="true" \ | |
| -e NEXT_PUBLIC_AXIOM_TOKEN="${{ secrets.NEXT_PUBLIC_AXIOM_TOKEN }}" \ | |
| -e NEXT_PUBLIC_AXIOM_DATASET="${{ secrets.NEXT_PUBLIC_AXIOM_DATASET }}" \ | |
| ${{ env.DOCKER_IMAGE }} | |
| # Wait for server to be ready | |
| echo "Waiting for app server to start..." | |
| SERVER_READY=false | |
| for i in {1..60}; do | |
| if curl -sf http://localhost:3000 > /dev/null 2>&1; then | |
| echo "App server is ready" | |
| SERVER_READY=true | |
| break | |
| fi | |
| sleep 2 | |
| done | |
| if [ "$SERVER_READY" != "true" ]; then | |
| echo "ERROR: App server failed to start within 120 seconds" | |
| docker logs inbox-zero-e2e | |
| exit 1 | |
| fi | |
| - name: Start ngrok tunnel | |
| run: | | |
| ngrok http 3000 --log=stdout > ngrok.log 2>&1 & | |
| sleep 5 | |
| # Extract the public URL | |
| NGROK_URL=$(curl -s http://localhost:4040/api/tunnels | jq -r '.tunnels[0].public_url') | |
| echo "NGROK_URL=$NGROK_URL" >> $GITHUB_ENV | |
| echo "Tunnel URL: $NGROK_URL" | |
| - name: Run E2E Flow Tests | |
| shell: bash | |
| run: | | |
| if [ -n "$E2E_FLOW_TEST_FILE" ]; then | |
| if [[ "$E2E_FLOW_TEST_FILE" == -* ]] || | |
| [[ "$E2E_FLOW_TEST_FILE" == /* ]] || | |
| [[ "$E2E_FLOW_TEST_FILE" == *..* ]] || | |
| ! [[ "$E2E_FLOW_TEST_FILE" =~ ^[A-Za-z0-9._/-]+$ ]]; then | |
| echo "::error::Invalid test_file input. Use a relative E2E flow test name or path containing only letters, numbers, dots, underscores, hyphens, and slashes." | |
| exit 1 | |
| fi | |
| pnpm -F inbox-zero-ai test-e2e:flows -- "$E2E_FLOW_TEST_FILE" | |
| else | |
| pnpm -F inbox-zero-ai test-e2e:flows | |
| fi | |
| env: | |
| E2E_FLOW_TEST_FILE: ${{ github.event.inputs.test_file }} | |
| NEXT_PUBLIC_BASE_URL: ${{ env.NGROK_URL }} | |
| # Test control | |
| RUN_E2E_FLOW_TESTS: "true" | |
| E2E_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} | |
| NEXT_PUBLIC_BYPASS_PREMIUM_CHECKS: ${{ secrets.NEXT_PUBLIC_BYPASS_PREMIUM_CHECKS }} | |
| NEXT_PUBLIC_IS_RESEND_CONFIGURED: ${{ secrets.NEXT_PUBLIC_IS_RESEND_CONFIGURED }} | |
| DISABLE_LOG_ZOD_ERRORS: ${{ secrets.DISABLE_LOG_ZOD_ERRORS }} | |
| # E2E-specific: Test account emails | |
| E2E_GMAIL_EMAIL: ${{ secrets.E2E_GMAIL_EMAIL }} | |
| E2E_OUTLOOK_EMAIL: ${{ secrets.E2E_OUTLOOK_EMAIL }} | |
| # Standard app secrets (reused from existing config) | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| DIRECT_URL: ${{ secrets.DIRECT_URL }} | |
| UPSTASH_REDIS_URL: ${{ secrets.UPSTASH_REDIS_URL }} | |
| UPSTASH_REDIS_TOKEN: ${{ secrets.UPSTASH_REDIS_TOKEN }} | |
| UPSTASH_REDIS_REST_URL: ${{ secrets.UPSTASH_REDIS_REST_URL }} | |
| UPSTASH_REDIS_REST_TOKEN: ${{ secrets.UPSTASH_REDIS_REST_TOKEN }} | |
| REDIS_URL: ${{ secrets.REDIS_URL }} | |
| GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} | |
| GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} | |
| GOOGLE_PUBSUB_TOPIC_NAME: ${{ secrets.GOOGLE_PUBSUB_TOPIC_NAME }} | |
| GOOGLE_PUBSUB_VERIFICATION_TOKEN: ${{ secrets.GOOGLE_PUBSUB_VERIFICATION_TOKEN }} | |
| MICROSOFT_CLIENT_ID: ${{ secrets.MICROSOFT_CLIENT_ID }} | |
| MICROSOFT_CLIENT_SECRET: ${{ secrets.MICROSOFT_CLIENT_SECRET }} | |
| MICROSOFT_WEBHOOK_CLIENT_STATE: ${{ secrets.MICROSOFT_WEBHOOK_CLIENT_STATE }} | |
| # AI provider secrets - configure whichever provider you use | |
| DEFAULT_LLM_PROVIDER: ${{ secrets.DEFAULT_LLM_PROVIDER }} | |
| DEFAULT_LLM_MODEL: ${{ secrets.DEFAULT_LLM_MODEL }} | |
| DEFAULT_OPENROUTER_PROVIDERS: ${{ secrets.DEFAULT_OPENROUTER_PROVIDERS }} | |
| ECONOMY_LLM_PROVIDER: ${{ secrets.ECONOMY_LLM_PROVIDER }} | |
| ECONOMY_LLM_MODEL: ${{ secrets.ECONOMY_LLM_MODEL }} | |
| ECONOMY_OPENROUTER_PROVIDERS: ${{ secrets.ECONOMY_OPENROUTER_PROVIDERS }} | |
| CHAT_LLM_PROVIDER: ${{ secrets.CHAT_LLM_PROVIDER }} | |
| CHAT_LLM_MODEL: ${{ secrets.CHAT_LLM_MODEL }} | |
| CHAT_OPENROUTER_PROVIDERS: ${{ secrets.CHAT_OPENROUTER_PROVIDERS }} | |
| OPENROUTER_BACKUP_MODEL: ${{ secrets.OPENROUTER_BACKUP_MODEL }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} | |
| BEDROCK_ACCESS_KEY: ${{ secrets.BEDROCK_ACCESS_KEY }} | |
| BEDROCK_SECRET_KEY: ${{ secrets.BEDROCK_SECRET_KEY }} | |
| BEDROCK_REGION: ${{ secrets.BEDROCK_REGION }} | |
| OLLAMA_BASE_URL: ${{ secrets.OLLAMA_BASE_URL }} | |
| OLLAMA_MODEL: ${{ secrets.OLLAMA_MODEL }} | |
| AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} | |
| AUTH_SECRET: ${{ secrets.AUTH_SECRET }} | |
| EMAIL_ENCRYPT_SECRET: ${{ secrets.EMAIL_ENCRYPT_SECRET }} | |
| EMAIL_ENCRYPT_SALT: ${{ secrets.EMAIL_ENCRYPT_SALT }} | |
| INTERNAL_API_KEY: ${{ secrets.INTERNAL_API_KEY }} | |
| RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} | |
| TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_TOKEN }} | |
| TINYBIRD_BASE_URL: ${{ secrets.TINYBIRD_BASE_URL }} | |
| - name: Collect Docker logs | |
| if: always() | |
| run: | | |
| docker logs inbox-zero-e2e > docker-server.log 2>&1 || true | |
| - name: Upload test logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: e2e-flow-logs-${{ github.run_id }} | |
| path: | | |
| apps/web/__tests__/e2e/flows/*.log | |
| docker-server.log | |
| ngrok.log | |
| retention-days: 7 | |
| notify-disabled: | |
| needs: check-enabled | |
| if: needs.check-enabled.outputs.enabled != 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: E2E flows disabled notice | |
| run: | | |
| echo "::notice::E2E flow tests are disabled. To enable, set the repository variable E2E_FLOWS_ENABLED=true" | |
| echo "" | |
| echo "Required secrets for E2E flow tests:" | |
| echo "" | |
| echo "E2E-specific secrets:" | |
| echo " - E2E_GMAIL_EMAIL: Gmail test account email" | |
| echo " - E2E_OUTLOOK_EMAIL: Outlook test account email" | |
| echo " - E2E_NGROK_AUTH_TOKEN: ngrok auth token for tunnel" | |
| echo "" | |
| echo "Standard app secrets (same as production):" | |
| echo " - DATABASE_URL, AUTH_SECRET, INTERNAL_API_KEY" | |
| echo " - EMAIL_ENCRYPT_SECRET, EMAIL_ENCRYPT_SALT" | |
| echo " - UPSTASH_REDIS_REST_URL, UPSTASH_REDIS_REST_TOKEN" | |
| echo " - GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET" | |
| echo " - GOOGLE_PUBSUB_TOPIC_NAME, GOOGLE_PUBSUB_VERIFICATION_TOKEN" | |
| echo " - MICROSOFT_CLIENT_ID, MICROSOFT_CLIENT_SECRET, MICROSOFT_WEBHOOK_CLIENT_STATE" | |
| echo " - AI provider secrets (one of: OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY, OPENROUTER_API_KEY)" | |
| echo " - DEFAULT_LLM_PROVIDER (optional, defaults to openai)" |