Nightly E2E Test Suite #74
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly E2E Test Suite | |
| on: | |
| schedule: | |
| # Run at 6:00 AM UTC Monday-Friday | |
| - cron: '0 6 * * 1-5' | |
| push: | |
| branches: | |
| - nightly/* | |
| workflow_dispatch: # Allow manual triggering | |
| jobs: | |
| nightly-e2e-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| cache: "yarn" | |
| cache-dependency-path: "yarn.lock" | |
| - name: Enable Corepack | |
| run: corepack enable | |
| - name: Install dependencies | |
| run: yarn install | |
| - name: Setup Supabase CLI | |
| uses: supabase/setup-cli@v1 | |
| with: | |
| version: latest | |
| - name: Start Clickhouse and Minio | |
| run: cd docker && docker compose up minio minio-setup clickhouse -d | |
| - name: Start Supabase | |
| run: | | |
| echo 'y' | npx supabase start -x realtime,storage-api,imgproxy,mailpit,edge-runtime,logflare,vector,supavisor | |
| - name: Start Gateway Workers | |
| run: | | |
| cd worker | |
| yarn install | |
| mkdir -p logs | |
| echo "Starting HELICONE_API worker on port 8788..." | |
| npx wrangler dev --var WORKER_TYPE:HELICONE_API --port 8788 --inspector-port=9240 > logs/wrangler-helicone-api.log 2>&1 & | |
| echo $! > logs/wrangler-helicone-api.pid | |
| echo "Starting AI_GATEWAY_API worker on port 8793..." | |
| npx wrangler dev --var WORKER_TYPE:AI_GATEWAY_API --var HELICONE_ORG_ID:"a75d76e3-02e7-4d02-8a2b-c65ed27c69b2" --port 8793 --inspector-port=9241 > logs/wrangler-ai-gateway.log 2>&1 & | |
| echo $! > logs/wrangler-ai-gateway.pid | |
| echo "Waiting for workers to start..." | |
| sleep 10 | |
| - name: Health Check Workers | |
| run: | | |
| echo "Checking if workers are running..." | |
| MAX_RETRIES=30 | |
| RETRY_DELAY=2 | |
| # Check HELICONE_API worker (port 8788) | |
| for i in $(seq 1 $MAX_RETRIES); do | |
| if curl -f http://localhost:8788/healthcheck 2>/dev/null; then | |
| echo "✓ HELICONE_API worker is running on port 8788" | |
| break | |
| fi | |
| if [ $i -eq $MAX_RETRIES ]; then | |
| echo "✗ HELICONE_API worker failed to start on port 8788" | |
| echo "Last 50 lines of wrangler-helicone-api.log:" | |
| tail -50 worker/logs/wrangler-helicone-api.log || true | |
| exit 1 | |
| fi | |
| echo "Waiting for HELICONE_API worker... (attempt $i/$MAX_RETRIES)" | |
| sleep $RETRY_DELAY | |
| done | |
| # Check AI_GATEWAY_API worker (port 8793) | |
| for i in $(seq 1 $MAX_RETRIES); do | |
| if curl -f http://localhost:8793/healthcheck 2>/dev/null; then | |
| echo "✓ AI_GATEWAY_API worker is running on port 8793" | |
| break | |
| fi | |
| if [ $i -eq $MAX_RETRIES ]; then | |
| echo "✗ AI_GATEWAY_API worker failed to start on port 8793" | |
| echo "Last 50 lines of wrangler-ai-gateway.log:" | |
| tail -50 worker/logs/wrangler-ai-gateway.log || true | |
| exit 1 | |
| fi | |
| echo "Waiting for AI_GATEWAY_API worker... (attempt $i/$MAX_RETRIES)" | |
| sleep $RETRY_DELAY | |
| done | |
| echo "All workers are running!" | |
| - name: Start Jawn | |
| run: | | |
| cd valhalla/jawn | |
| cp .env.hosted.example .env | |
| yarn install | |
| yarn dev & | |
| sleep 10 | |
| - name: Setup Python for populate-keys | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Populate Provider API Keys | |
| env: | |
| PRODUCTION_PROVIDER_API_KEYS: ${{ secrets.PRODUCTION_PROVIDER_API_KEYS }} | |
| run: | | |
| pip install requests | |
| python scripts/populate-keys/main.py | |
| - name: Run E2E Tests | |
| run: | | |
| cd e2e | |
| yarn install | |
| # Run regular tests (excluding nightly) | |
| yarn test | |
| # Run nightly tests explicitly | |
| yarn test tests/nightly | |
| - name: Display Logs on Failure | |
| if: failure() | |
| run: | | |
| echo "=========================================" | |
| echo "HELICONE_API Worker Logs (last 100 lines)" | |
| echo "=========================================" | |
| tail -100 worker/logs/wrangler-helicone-api.log || echo "No logs found for HELICONE_API worker" | |
| echo "" | |
| echo "=========================================" | |
| echo "AI_GATEWAY_API Worker Logs (last 100 lines)" | |
| echo "=========================================" | |
| tail -100 worker/logs/wrangler-ai-gateway.log || echo "No logs found for AI_GATEWAY_API worker" | |
| echo "" | |
| echo "=========================================" | |
| echo "Worker Process Status" | |
| echo "=========================================" | |
| ps aux | grep wrangler || echo "No wrangler processes found" | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| pkill -f wrangler || true | |
| pkill -f "yarn dev" || true | |
| supabase stop --no-backup || true | |
| cd docker && docker compose down || true |