e2e_tests_providers.yaml

# .github/workflows/e2e_tests.yml
name: E2E Inference Provider Tests

on: 
  schedule:
    - cron: "0 0 * * *"  # Runs once a day at midnight UTC
  workflow_dispatch:

jobs:
  e2e_tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        mode: ["server", "library"]
        environment: ["azure", "vertexai", "watsonx"]
    
    name: "E2E: ${{ matrix.mode }} mode / ${{ matrix.environment }}"
    
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      E2E_OPENAI_MODEL: ${{ vars.E2E_OPENAI_MODEL }}
      CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
      CLIENT_ID: ${{ secrets.CLIENT_ID }}
      TENANT_ID: ${{ secrets.TENANT_ID }}
      E2E_DEPLOYMENT_MODE: ${{ matrix.mode }}
      FAISS_VECTOR_STORE_ID: ${{ vars.FAISS_VECTOR_STORE_ID }}

    steps:
      - uses: actions/checkout@v4
        with:
          # On PR_TARGET → the fork (or same repo) that opened the PR.
          # On push      → falls back to the current repository.
          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

          # On PR_TARGET → the PR head *commit* (reproducible).
          # On push      → the pushed commit that triggered the workflow.
          ref: ${{ github.event.pull_request.head.ref || github.sha }}

          # Don’t keep credentials when running untrusted PR code under PR_TARGET.
          persist-credentials: ${{ github.event_name != 'pull_request_target' }}

      - name: Verify actual git checkout result
        run: |
          echo "=== Git Status After Checkout ==="
          echo "Remote URLs:"
          git remote -v
          echo ""
          echo "Current branch: $(git branch --show-current 2>/dev/null || echo 'detached HEAD')"
          echo "Current commit: $(git rev-parse HEAD)"
          echo "Current commit message: $(git log -1 --oneline)"
          echo ""
          echo "=== Recent commits ==="
          git log --oneline -5

      - name: Add Azure Entra ID config block to all test configs
        if: matrix.environment == 'azure'
        run: |
          echo "Adding azure_entra_id configuration block to all test configs..."
          for config in tests/e2e/configuration/*/lightspeed-stack*.yaml; do
            if [ -f "$config" ]; then
              echo "" >> "$config"
              echo "azure_entra_id:" >> "$config"
              echo "  tenant_id: \${env.TENANT_ID}" >> "$config"
              echo "  client_id: \${env.CLIENT_ID}" >> "$config"
              echo "  client_secret: \${env.CLIENT_SECRET}" >> "$config"
              echo "✅ Added to: $config"
            fi
          done

      - name: Load lightspeed-stack.yaml configuration
        run: |
          MODE="${{ matrix.mode }}"
          CONFIG_FILE="tests/e2e/configuration/${MODE}-mode/lightspeed-stack.yaml"
          echo "Loading configuration for ${MODE} mode"
          echo "Source: ${CONFIG_FILE}"
          
          if [ ! -f "${CONFIG_FILE}" ]; then
            echo "❌ Configuration file not found: ${CONFIG_FILE}"
            exit 1
          fi
          
          cp "${CONFIG_FILE}" lightspeed-stack.yaml
          echo "✅ Configuration loaded successfully"

      - name: Save VertexAI service account key to file
        if: matrix.environment == 'vertexai'
        env:
          GOOGLE_SA_KEY: ${{ secrets.GOOGLE_SA_KEY }}
        run: |
          echo "Setting up Google Cloud service account credentials..."

          if [ -z "$GOOGLE_SA_KEY" ]; then
            echo "❌ GOOGLE_SA_KEY is not set. Please configure the secret in GitHub repository settings."
            exit 1
          fi

          GCP_KEYS_PATH=./tmp/.gcp-keys
          echo "GCP_KEYS_PATH=$GCP_KEYS_PATH" >> $GITHUB_ENV

          mkdir -p $GCP_KEYS_PATH

          echo "Writing service account key to file..."

          # Decode from base64, needed because GH changes the key if using the raw key
          printf '%s' "$GOOGLE_SA_KEY" | base64 -d > $GCP_KEYS_PATH/gcp-key.json

          # Verify the file was created and is valid JSON
          if [ ! -f "$GCP_KEYS_PATH/gcp-key.json" ]; then
            echo "❌ Failed to create gcp-key.json file"
            exit 1
          fi

          if ! jq empty "$GCP_KEYS_PATH/gcp-key.json" 2>/dev/null; then
            echo "❌ gcp-key.json is not valid JSON"
            exit 1
          fi
          echo "✅ gcp-key.json is valid JSON"

          # Set proper permissions (readable by all, needed for container user 1001)
          chmod 644 $GCP_KEYS_PATH/gcp-key.json

          echo "GOOGLE_APPLICATION_CREDENTIALS=/opt/app-root/.gcp-keys/gcp-key.json" >> $GITHUB_ENV

      - name: Select and configure run.yaml
        env:
          CONFIG_ENVIRONMENT: ${{ matrix.environment || 'ci' }}
        run: |
          CONFIGS_DIR="tests/e2e/configs"
          ENVIRONMENT="$CONFIG_ENVIRONMENT"
          
          echo "Looking for configurations in $CONFIGS_DIR/"
          
          # List available configurations
          if [ -d "$CONFIGS_DIR" ]; then
            echo "Available configurations:"
            ls -la "$CONFIGS_DIR"/*.yaml 2>/dev/null || echo "No YAML files found in $CONFIGS_DIR/"
          else
            echo "Configs directory '$CONFIGS_DIR' not found!"
            exit 1
          fi
          
          # Determine which config file to use
          CONFIG_FILE="$CONFIGS_DIR/run-$ENVIRONMENT.yaml"
          
          echo "Looking for: $CONFIG_FILE"
          
          if [ -f "$CONFIG_FILE" ]; then
            echo "✅ Found config for environment: $ENVIRONMENT"
            cp "$CONFIG_FILE" run.yaml
            echo "✅ Config copied to run.yaml"
          else
            echo "❌ Configuration file not found: $CONFIG_FILE"
            echo "Available files in $CONFIGS_DIR:"
            ls -la "$CONFIGS_DIR/"
            exit 1
          fi

      - name: Show final configuration
        run: |
          echo "=== Configuration Summary ==="
          echo "Deployment mode: ${{ matrix.mode }}"
          echo "Environment: ${{ matrix.environment }}"
          echo "Source config: tests/e2e/configs/run-ci.yaml"
          echo ""
          echo "=== Configuration Preview ==="
          echo "Providers: $(grep -c "provider_id:" run.yaml)"
          echo "Models: $(grep -c "model_id:" run.yaml)"
          echo ""
          echo "=== lightspeed-stack.yaml ==="
          grep -A 3 "llama_stack:" lightspeed-stack.yaml

      - name: Docker Login for quay access
        env:
          QUAY_ROBOT_USERNAME: ${{ secrets.QUAY_DOWNSTREAM_USERNAME }}
          QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_DOWNSTREAM_TOKEN }}
        run: |
          echo $QUAY_ROBOT_TOKEN | docker login quay.io -u=$QUAY_ROBOT_USERNAME --password-stdin

      - name: Create dummy GCP keys directory
        if: matrix.environment != 'vertexai'
        run: |
          echo "Creating dummy GCP keys directory for non-VertexAI environment..."
          mkdir -p ./tmp/.gcp-keys-dummy
          echo "✅ Dummy directory created."

      - name: Run services (Server Mode)
        if: matrix.mode == 'server'
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          TENANT_ID: ${{ secrets.TENANT_ID }}
          CLIENT_ID: ${{ secrets.CLIENT_ID }}
          CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
          VERTEX_AI_LOCATION: ${{ secrets.VERTEX_AI_LOCATION }}
          VERTEX_AI_PROJECT: ${{ secrets.VERTEX_AI_PROJECT }}
          GOOGLE_APPLICATION_CREDENTIALS: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
          GCP_KEYS_PATH: ${{ env.GCP_KEYS_PATH }}
          WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
          WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }}

        run: |
          # Debug: Check if environment variable is available for docker-compose
          echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"
          echo "OPENAI_API_KEY length: ${#OPENAI_API_KEY}"

          docker compose version
          docker compose up -d

          # Check for errors and show logs if any services failed
          if docker compose ps | grep -E 'Exit|exited|stopped'; then
            echo "Some services failed to start - showing logs:"
            docker compose logs
            exit 1
          else
            echo "All services started successfully"
          fi

      - name: Run services (Library Mode)
        if: matrix.mode == 'library'
        env: 
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          TENANT_ID: ${{ secrets.TENANT_ID }}
          CLIENT_ID: ${{ secrets.CLIENT_ID }}
          CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
          VERTEX_AI_LOCATION: ${{ secrets.VERTEX_AI_LOCATION }}
          VERTEX_AI_PROJECT: ${{ secrets.VERTEX_AI_PROJECT }}
          GOOGLE_APPLICATION_CREDENTIALS: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
          GCP_KEYS_PATH: ${{ env.GCP_KEYS_PATH }}
          WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
          WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }}

        run: |
          echo "Starting service in library mode (1 container)"
          docker compose -f docker-compose-library.yaml up -d

          if docker compose -f docker-compose-library.yaml ps | grep -E 'Exit|exited|stopped'; then
            echo "Service failed to start - showing logs:"
            docker compose -f docker-compose-library.yaml logs
            exit 1
          else
            echo "Service started successfully"
          fi

      - name: Wait for services
        run: |
          echo "Waiting for services to be healthy..."
          sleep 20

      - name: Quick connectivity test
        run: |
          echo "Testing basic connectivity before full test suite..."
          curl -f http://localhost:8080/v1/models || {
            echo "❌ Basic connectivity failed - showing logs"
            if [ "${{ matrix.mode }}" == "server" ]; then
              docker compose logs --tail=30
            else
              docker compose -f docker-compose-library.yaml logs --tail=30
            fi
            exit 1
          }

      # watsonx has a different convention than "<provider>/<model>"
      - name: Set watsonx test overrides
        if: matrix.environment == 'watsonx'
        run: |
          echo "E2E_DEFAULT_MODEL_OVERRIDE=watsonx/watsonx/meta-llama/llama-3-3-70b-instruct" >> $GITHUB_ENV
          echo "E2E_DEFAULT_PROVIDER_OVERRIDE=watsonx" >> $GITHUB_ENV

      - name: Run e2e tests
        env:
          TERM: xterm-256color
          FORCE_COLOR: 1
          E2E_DEPLOYMENT_MODE: ${{ matrix.mode }}
        run: |
          echo "Installing test dependencies..."
          pip install uv
          uv sync

          echo "Running comprehensive e2e test suite..."
          make test-e2e

      - name: Show logs on failure
        if: failure()
        run: |
          echo "=== Test failure logs ==="
          
          if [ "${{ matrix.mode }}" == "server" ]; then
            echo "=== llama-stack logs ==="
            docker compose logs llama-stack
            echo ""
            echo "=== lightspeed-stack logs ==="
            docker compose logs lightspeed-stack
          else
            echo "=== lightspeed-stack (library mode) logs ==="
            docker compose -f docker-compose-library.yaml logs lightspeed-stack
          fi