nightly-e2e-test-suite.yml

name: Nightly E2E Test Suite

on:
  schedule:
    # Run at 6:00 AM UTC Monday-Friday
    - cron: '0 6 * * 1-5'
  push:
    branches:
      - nightly/*
  workflow_dispatch:  # Allow manual triggering

jobs:
  nightly-e2e-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 20

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "20"
          cache: "yarn"
          cache-dependency-path: "yarn.lock"

      - name: Enable Corepack
        run: corepack enable

      - name: Install dependencies
        run: yarn install

      - name: Setup Supabase CLI
        uses: supabase/setup-cli@v1
        with:
          version: latest

      - name: Start Clickhouse and Minio
        run: cd docker && docker compose up minio minio-setup clickhouse -d

      - name: Start Supabase
        run: |
          echo 'y' | npx supabase start -x realtime,storage-api,imgproxy,mailpit,edge-runtime,logflare,vector,supavisor

      - name: Start Gateway Workers
        run: |
          cd worker
          yarn install
          mkdir -p logs
          echo "Starting HELICONE_API worker on port 8788..."
          npx wrangler dev --var WORKER_TYPE:HELICONE_API --port 8788 --inspector-port=9240 > logs/wrangler-helicone-api.log 2>&1 &
          echo $! > logs/wrangler-helicone-api.pid
          echo "Starting AI_GATEWAY_API worker on port 8793..."
          npx wrangler dev --var WORKER_TYPE:AI_GATEWAY_API --var HELICONE_ORG_ID:"a75d76e3-02e7-4d02-8a2b-c65ed27c69b2" --port 8793 --inspector-port=9241 > logs/wrangler-ai-gateway.log 2>&1 &
          echo $! > logs/wrangler-ai-gateway.pid
          echo "Waiting for workers to start..."
          sleep 10

      - name: Health Check Workers
        run: |
          echo "Checking if workers are running..."
          MAX_RETRIES=30
          RETRY_DELAY=2

          # Check HELICONE_API worker (port 8788)
          for i in $(seq 1 $MAX_RETRIES); do
            if curl -f http://localhost:8788/healthcheck 2>/dev/null; then
              echo "✓ HELICONE_API worker is running on port 8788"
              break
            fi
            if [ $i -eq $MAX_RETRIES ]; then
              echo "✗ HELICONE_API worker failed to start on port 8788"
              echo "Last 50 lines of wrangler-helicone-api.log:"
              tail -50 worker/logs/wrangler-helicone-api.log || true
              exit 1
            fi
            echo "Waiting for HELICONE_API worker... (attempt $i/$MAX_RETRIES)"
            sleep $RETRY_DELAY
          done

          # Check AI_GATEWAY_API worker (port 8793)
          for i in $(seq 1 $MAX_RETRIES); do
            if curl -f http://localhost:8793/healthcheck 2>/dev/null; then
              echo "✓ AI_GATEWAY_API worker is running on port 8793"
              break
            fi
            if [ $i -eq $MAX_RETRIES ]; then
              echo "✗ AI_GATEWAY_API worker failed to start on port 8793"
              echo "Last 50 lines of wrangler-ai-gateway.log:"
              tail -50 worker/logs/wrangler-ai-gateway.log || true
              exit 1
            fi
            echo "Waiting for AI_GATEWAY_API worker... (attempt $i/$MAX_RETRIES)"
            sleep $RETRY_DELAY
          done

          echo "All workers are running!"

      - name: Start Jawn
        run: |
          cd valhalla/jawn
          cp .env.hosted.example .env
          yarn install
          yarn dev &
          sleep 10

      - name: Setup Python for populate-keys
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'

      - name: Populate Provider API Keys
        env:
          PRODUCTION_PROVIDER_API_KEYS: ${{ secrets.PRODUCTION_PROVIDER_API_KEYS }}
        run: |
          pip install requests
          python scripts/populate-keys/main.py

      - name: Run E2E Tests
        run: |
          cd e2e
          yarn install
          # Run regular tests (excluding nightly)
          yarn test
          # Run nightly tests explicitly
          yarn test tests/nightly

      - name: Display Logs on Failure
        if: failure()
        run: |
          echo "========================================="
          echo "HELICONE_API Worker Logs (last 100 lines)"
          echo "========================================="
          tail -100 worker/logs/wrangler-helicone-api.log || echo "No logs found for HELICONE_API worker"
          echo ""
          echo "========================================="
          echo "AI_GATEWAY_API Worker Logs (last 100 lines)"
          echo "========================================="
          tail -100 worker/logs/wrangler-ai-gateway.log || echo "No logs found for AI_GATEWAY_API worker"
          echo ""
          echo "========================================="
          echo "Worker Process Status"
          echo "========================================="
          ps aux | grep wrangler || echo "No wrangler processes found"

      - name: Cleanup
        if: always()
        run: |
          pkill -f wrangler || true
          pkill -f "yarn dev" || true
          supabase stop --no-backup || true
          cd docker && docker compose down || true