cypress-e2e-test.yml

name: Cypress e2e Test

# =============================================================================
# E2E Test Workflow with Cluster Failover and Smart Test Selection
# =============================================================================
#
# TRIGGERS:
#   - Automatically after "Test" workflow completes on PRs
#   - Manually via workflow_dispatch (Actions tab → Run workflow)
#
# CLUSTER FAILOVER:
#   Primary:   dash-e2e-int (checked first via DSC health)
#   Secondary: dash-e2e     (used if primary is unhealthy)
#   Health Check: Logs into cluster → checks DSC conditions (Available, Degraded, odh-dashboardReady)
#
# TEST SELECTION (priority order):
#   Default (always run):
#     - @ci-dashboard-regression-tags
#
#   1. Manual input (workflow_dispatch):
#      Enter tags in 'additional_tags' field: @Pipelines,@Workbenches
#
#   2. PR labels (test:* pattern):
#      Add labels with 'test:' prefix to your PR:
#        test:Pipelines    → @Pipelines
#        test:ModelServing → @ModelServing
#        test:Workbenches  → @Workbenches
#      Any 'test:<TagName>' label maps to '@<TagName>' Cypress grep tag
#
#   3. Auto-detected from PR changes (always additive):
#      Turbo detects changed packages → reads "e2eCiTags" from package.json
#      Git diff detects changed frontend sub-areas → inline mapping resolves tags
#      All auto-detected tags are consolidated into ONE additional matrix job
#
#      To add auto-detection for a package:
#        Add "e2eCiTags": ["@YourTagCI"] to the package's package.json
#      To add auto-detection for a frontend area:
#        Add an entry to .github/frontend-ci-tags.json
#
# BFF SUPPORT (Backend-For-Frontend):
#   Packages with bffConfig.enabled=true in their package.json are automatically
#   detected and started when changes are found. BFFs start after the frontend
#   webpack server. See docs/bff-e2e-testing.md for details.
#
#   To add BFF support to a package:
#     Add "bffConfig" to package.json with enabled, port, healthEndpoint,
#     startCommand, and startCommandCluster properties
#
# LIMITS:
#   - Max 5 additional tags for labels/manual (prevents runner exhaustion)
#   - Auto-detected tags are consolidated into 1 job (no limit needed)
#   - 10 runners shared across 30+ devs
#
# REQUIRED SECRETS:
#   PRIMARY:   OC_SERVER_PRIMARY, OCP_CONSOLE_URL_PRIMARY, ODH_DASHBOARD_URL_PRIMARY
#   SECONDARY: OC_SERVER, OCP_CONSOLE_URL, ODH_DASHBOARD_URL
#   AUTH:      GITLAB_TOKEN, GITLAB_TEST_VARS_URL, ODH_NAMESPACES
# =============================================================================

on:
  workflow_run:
    workflows: ["Test"]
    types: [completed]

  workflow_dispatch:
    inputs:
      additional_tags:
        description: 'Extra test tags (e.g., @Pipelines,@Workbenches)'
        required: false
        default: ''
        type: string

concurrency:
  group: e2e-${{ github.event.workflow_run.head_branch || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  pull-requests: read
  actions: read
  statuses: write

env:
  NODE_VERSION: 22.x
  DO_NOT_TRACK: 1

# =============================================================================
# JOBS
# =============================================================================
jobs:

  # ---------------------------------------------------------------------------
  # Cluster Selection - Health check with automatic failover
  # ---------------------------------------------------------------------------
  select-cluster:
    if: >-
      github.event_name == 'workflow_dispatch' || 
      (github.event.workflow_run.event == 'pull_request' && 
       github.event.workflow_run.conclusion == 'success')
    runs-on: self-hosted
    outputs:
      cluster_name: ${{ steps.select.outputs.cluster_name }}
    steps:
      - name: Download test credentials
        run: |
          echo "🔧 Downloading test credentials for cluster health check..."
          curl -fk -H "Authorization: Bearer ${{ secrets.GITLAB_TOKEN }}" \
            "${{ secrets.GITLAB_TEST_VARS_URL }}" \
            -o /tmp/test-variables.yml
          echo "✅ Downloaded test credentials"

      - name: Select healthy cluster
        id: select
        env:
          PRIMARY_SERVER: ${{ secrets.OC_SERVER_PRIMARY }}
          PRIMARY_DASHBOARD: ${{ secrets.ODH_DASHBOARD_URL_PRIMARY }}
          SECONDARY_SERVER: ${{ secrets.OC_SERVER }}
          SECONDARY_DASHBOARD: ${{ secrets.ODH_DASHBOARD_URL }}
        run: |
          # Extract credentials from test-variables.yml
          TEST_VARS_FILE="/tmp/test-variables.yml"
          OC_USERNAME=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "USERNAME:" | head -1 | sed 's/.*USERNAME: //' | tr -d ' ')
          OC_PASSWORD=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "PASSWORD:" | head -1 | sed 's/.*PASSWORD: //' | tr -d ' ')
          echo "::add-mask::$OC_PASSWORD"
          echo "::add-mask::$OC_USERNAME"

          # Check DSC health by logging in and verifying conditions
          check_dsc_health() {
            local server_url="$1"
            local cluster_name="$2"
            
            [[ -z "$server_url" ]] && echo "  ❌ Server URL is empty" && return 1
            
            echo "  🔗 Attempting login to: $server_url"
            
            # Try to login
            LOGIN_OUTPUT=$(oc login -u "$OC_USERNAME" -p "$OC_PASSWORD" --server="$server_url" --insecure-skip-tls-verify 2>&1) || true
            if ! oc whoami > /dev/null 2>&1; then
              echo "  ❌ Failed to login to $cluster_name"
              echo "  📝 Login output: $LOGIN_OUTPUT" | head -5
              return 1
            fi
            echo "  ✅ Login successful"
            
            # Get DSC status with full output for debugging
            echo "  🔍 Fetching DataScienceCluster status..."
            DSC_JSON=$(oc get datasciencecluster -o json 2>&1)
            DSC_EXIT_CODE=$?
            
            if [[ $DSC_EXIT_CODE -ne 0 ]]; then
              echo "  ❌ Failed to get DSC (exit code: $DSC_EXIT_CODE)"
              echo "  📝 Output: $DSC_JSON" | head -5
              return 1
            fi
            
            if [[ -z "$DSC_JSON" || "$DSC_JSON" == "null" || "$DSC_JSON" == '{"apiVersion":"datasciencecluster.opendatahub.io/v1","items":[],"kind":"List","metadata":{"resourceVersion":""}}' ]]; then
              echo "  ❌ No DataScienceCluster found on $cluster_name"
              return 1
            fi
            
            # Print DSC name and status for debugging
            DSC_NAME=$(echo "$DSC_JSON" | jq -r '.items[0].metadata.name // "unknown"')
            echo "  📦 DSC Name: $DSC_NAME"
            
            # Check phase - this is the most reliable indicator
            PHASE=$(echo "$DSC_JSON" | jq -r '.items[0].status.phase // "Unknown"')
            echo "  📊 DSC Phase: $PHASE"
            
            # Print all conditions for debugging
            echo "  📋 DSC Conditions:"
            echo "$DSC_JSON" | jq -r '.items[0].status.conditions[]? | "     - \(.type): \(.status) (\(.reason // "no reason"))"' 2>/dev/null || echo "     (no conditions found)"
            
            # If phase is Ready, cluster is healthy
            if [[ "$PHASE" == "Ready" ]]; then
              echo "  ✅ DSC is Ready!"
              return 0
            fi
            
            # Phase not Ready - check conditions for more detail
            AVAILABLE=$(echo "$DSC_JSON" | jq -r '.items[0].status.conditions[] | select(.type=="Available") | .status' 2>/dev/null || echo "")
            DEGRADED=$(echo "$DSC_JSON" | jq -r '.items[0].status.conditions[] | select(.type=="Degraded") | .status' 2>/dev/null || echo "")
            
            # Fallback: if conditions show healthy even though phase isn't Ready
            if [[ "$AVAILABLE" == "True" && "$DEGRADED" != "True" ]]; then
              echo "  ✅ Conditions look healthy despite phase=$PHASE"
              return 0
            fi
            
            echo "  ❌ DSC not healthy (Phase: $PHASE, Available: $AVAILABLE, Degraded: $DEGRADED)"
            return 1
          }

          echo "🔍 Checking PRIMARY cluster (dash-e2e-int)..."
          if check_dsc_health "$PRIMARY_SERVER" "dash-e2e-int"; then
            echo "✅ PRIMARY cluster is healthy and ready"
            echo "cluster_name=dash-e2e-int" >> $GITHUB_OUTPUT
          else
            echo ""
            echo "⚠️ PRIMARY unavailable or not ready, trying SECONDARY (dash-e2e)..."
            if check_dsc_health "$SECONDARY_SERVER" "dash-e2e"; then
              echo "✅ SECONDARY cluster is healthy and ready"
              echo "cluster_name=dash-e2e" >> $GITHUB_OUTPUT
            else
              echo ""
              echo "❌ All clusters unavailable or unhealthy"
              exit 1
            fi
          fi
          
          # Clean up credentials file
          rm -f /tmp/test-variables.yml

  # ---------------------------------------------------------------------------
  # Status - Set pending status on PR (independent - runs before cluster selection)
  # ---------------------------------------------------------------------------
  set-pending-status:
    if: >-
      github.event_name == 'workflow_dispatch' || 
      (github.event.workflow_run.event == 'pull_request' && 
       github.event.workflow_run.conclusion == 'success')
    runs-on: ubuntu-latest
    steps:
      - name: Set pending status
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gh api repos/${{ github.repository }}/statuses/${{ github.event.workflow_run.head_sha || github.sha }} \
            -f state=pending \
            -f context="Cypress E2E Tests" \
            -f description="E2E tests starting..." \
            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  # ---------------------------------------------------------------------------
  # Tag Resolution - Build test matrix from defaults + PR labels/input + auto-detection
  # ---------------------------------------------------------------------------
  get-test-tags:
    needs: [select-cluster]
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.build.outputs.matrix }}
      source: ${{ steps.build.outputs.source }}
      bff_packages: ${{ steps.detect.outputs.bff_packages }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
          fetch-depth: 0

      - name: Setup Node.js ${{ env.NODE_VERSION }}
        uses: actions/setup-node@v4.3.0
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Get PR labels
        id: labels
        if: github.event_name == 'workflow_run'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          # Get PR number - try multiple methods for fork PR compatibility
          PR_NUM="${{ github.event.workflow_run.pull_requests[0].number }}"
          
          # Method 2: commits API (works for same-repo PRs)
          if [[ -z "$PR_NUM" || "$PR_NUM" == "null" ]]; then
            PR_NUM=$(gh api "repos/${{ github.repository }}/commits/${{ github.event.workflow_run.head_sha }}/pulls" \
              --jq '.[0].number' 2>/dev/null || echo "")
          fi
          
          # Method 3: search API (works for fork PRs)
          if [[ -z "$PR_NUM" || "$PR_NUM" == "null" ]]; then
            PR_NUM=$(gh api "search/issues?q=repo:${{ github.repository }}+is:pr+is:open+sha:${{ github.event.workflow_run.head_sha }}" \
              --jq '.items[0].number' 2>/dev/null || echo "")
          fi

          if [[ -n "$PR_NUM" && "$PR_NUM" != "null" ]]; then
            LABELS=$(gh api "repos/${{ github.repository }}/issues/$PR_NUM/labels" \
              --jq '[.[].name] | join(",")' 2>/dev/null || echo "")
            echo "labels=$LABELS" >> $GITHUB_OUTPUT
            echo "📋 PR #$PR_NUM labels: $LABELS"
          else
            echo "⚠️ Could not find PR number for SHA ${{ github.event.workflow_run.head_sha }}"
          fi

      - name: Detect changed areas
        id: detect
        run: |
          # =================================================================
          # Smart Test Selection: Detect changed areas and resolve CI tags
          #
          # Layer 1: Turbo detects changed packages (including frontend "//" workspace)
          #          → reads e2eCiTags from each package.json (self-service, teams opt in)
          # Layer 2: For frontend changes detected by Layer 1, git diff identifies
          #          sub-areas → .github/frontend-ci-tags.json mapping resolves CI tags
          # =================================================================

          AUTO_TAGS=""

          # --- Layer 1: Turbo-based package detection ---
          echo "🔍 Running Turbo change detection..."

          # Determine base ref for comparison
          if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
            BASE_SHA="${{ github.event.workflow_run.pull_requests[0].base.sha || 'origin/main' }}"
          else
            BASE_SHA="origin/main"
          fi
          HEAD_SHA="${{ github.event.workflow_run.head_sha || github.sha }}"

          echo "  📌 Comparing $BASE_SHA...$HEAD_SHA"

          # Get changed packages from turbo (uses dependency graph)
          CHANGED_PACKAGES=$(npx turbo run lint --dry=json --filter="...[$BASE_SHA...$HEAD_SHA]" 2>/dev/null \
            | jq -r '.packages[]' 2>/dev/null || echo "")

          if [[ -n "$CHANGED_PACKAGES" ]]; then
            echo "  📦 Changed workspaces detected by Turbo:"
            echo "$CHANGED_PACKAGES" | while read -r pkg; do echo "     - $pkg"; done

            # For each changed package, check for e2eCiTags in its package.json
            for pkg_dir in packages/*/; do
              pkg_name=$(jq -r '.name // empty' "$pkg_dir/package.json" 2>/dev/null)
              if echo "$CHANGED_PACKAGES" | grep -qx "$pkg_name"; then
                ci_tags=$(jq -r '.e2eCiTags[]? // empty' "$pkg_dir/package.json" 2>/dev/null)
                if [[ -n "$ci_tags" ]]; then
                  for tag in $ci_tags; do
                    echo "  ✅ $pkg_name → $tag"
                    AUTO_TAGS="$AUTO_TAGS $tag"
                  done
                else
                  echo "  ⏭️  $pkg_name (no e2eCiTags — defaults only)"
                fi
              fi
            done
          else
            echo "  ℹ️  No package changes detected by Turbo"
          fi

          # --- Layer 2: Frontend sub-area detection ---
          # Turbo sees the entire frontend as one workspace. When it changes,
          # use git diff to identify which sub-areas were modified.
          # Note: Turbo reports the root/frontend workspace as "//"
          if echo "$CHANGED_PACKAGES" | grep -qx "//"; then
            echo ""
            echo "🔍 Frontend changed — detecting sub-areas via git diff..."

            # Load frontend directory → CI tag mapping from external JSON file
            # To add a new area: edit .github/frontend-ci-tags.json
            MAPPING_FILE=".github/frontend-ci-tags.json"
            if [[ ! -f "$MAPPING_FILE" ]]; then
              echo "  ⚠️  $MAPPING_FILE not found — skipping frontend sub-area detection"
            else
              echo "  📄 Loaded mappings from $MAPPING_FILE"

              # Get changed frontend files
              CHANGED_FILES=$(git diff --name-only "$BASE_SHA"..."$HEAD_SHA" -- frontend/src/ 2>/dev/null || echo "")

              if [[ -n "$CHANGED_FILES" ]]; then
                # Scan pages/, concepts/, api/, routes/ using the same mapping
                for src_dir in pages concepts api routes; do
                  DIRS=$(echo "$CHANGED_FILES" | grep "^frontend/src/$src_dir/" | \
                    sed "s|^frontend/src/$src_dir/||" | cut -d'/' -f1 | sort -u)

                  for dir in $DIRS; do
                    tag=$(jq -r --arg d "$dir" '.[$d] // empty' "$MAPPING_FILE")
                    if [[ -n "$tag" ]]; then
                      echo "  ✅ $src_dir/$dir → $tag"
                      AUTO_TAGS="$AUTO_TAGS $tag"
                    fi
                  done
                done
              fi
            fi
          fi

          # Deduplicate auto-detected tags
          if [[ -n "$AUTO_TAGS" ]]; then
            AUTO_TAGS=$(echo "$AUTO_TAGS" | tr ' ' '\n' | sort -u | tr '\n' ' ' | xargs)
            echo ""
            echo "🏷️  Auto-detected CI tags: $AUTO_TAGS"
          else
            echo ""
            echo "ℹ️  No area-specific CI tags detected — defaults only"
          fi

          echo "auto_tags=$AUTO_TAGS" >> $GITHUB_OUTPUT

          # --- BFF Package Detection ---
          # Detect changed packages that have bffConfig.enabled=true
          echo ""
          echo "🔍 Detecting BFF packages to start..."
          BFF_PACKAGES="[]"

          if [[ -n "$CHANGED_PACKAGES" ]]; then
            for pkg_dir in packages/*/; do
              pkg_name=$(jq -r '.name // empty' "$pkg_dir/package.json" 2>/dev/null)

              # Check if this package changed
              if echo "$CHANGED_PACKAGES" | grep -qx "$pkg_name"; then
                # Check if package has bffConfig with enabled=true
                bff_enabled=$(jq -r '.bffConfig.enabled // false' "$pkg_dir/package.json" 2>/dev/null)

                if [[ "$bff_enabled" == "true" ]]; then
                  bff_port=$(jq -r '.bffConfig.port // 4000' "$pkg_dir/package.json" 2>/dev/null)
                  bff_health=$(jq -r '.bffConfig.healthEndpoint // "/api/health"' "$pkg_dir/package.json" 2>/dev/null)
                  bff_cmd=$(jq -r '.bffConfig.startCommandCluster // .bffConfig.startCommand' "$pkg_dir/package.json" 2>/dev/null)
                  pkg_dir_name=$(basename "$pkg_dir")

                  echo "  ✅ $pkg_name (port: $bff_port, health: $bff_health)"

                  # Add to JSON array
                  BFF_PACKAGES=$(echo "$BFF_PACKAGES" | jq -c --arg name "$pkg_name" --arg dir "$pkg_dir_name" --arg port "$bff_port" --arg health "$bff_health" --arg cmd "$bff_cmd" '. + [{"name": $name, "dir": $dir, "port": ($port | tonumber), "healthEndpoint": $health, "startCommand": $cmd}]')
                fi
              fi
            done
          fi

          if [[ "$BFF_PACKAGES" == "[]" ]]; then
            echo "  ℹ️  No BFF packages detected for this change"
          fi

          echo "bff_packages=$BFF_PACKAGES" >> $GITHUB_OUTPUT

      - name: Build test matrix
        id: build
        run: |
          # Configuration
          MAX_EXTRA_TAGS=5  # Limit additional tags to prevent runner exhaustion (for labels/manual only)
          
          # Defaults - these ALWAYS run
          TAGS="@ci-dashboard-regression-tags"
          SOURCE="default"
          EXTRA_COUNT=0
          AUTO_DETECTED_ENTRY=""

          # Priority 1: Manual input (workflow_dispatch)
          if [[ -n "${{ inputs.additional_tags }}" ]]; then
            for tag in $(echo "${{ inputs.additional_tags }}" | tr ',' ' '); do
              if [[ $EXTRA_COUNT -lt $MAX_EXTRA_TAGS ]]; then
                TAGS="$TAGS,$tag"
                EXTRA_COUNT=$((EXTRA_COUNT + 1))
              fi
            done
            SOURCE="manual"
            echo "📝 Added manual tags (limit: $MAX_EXTRA_TAGS)"

          # Priority 2: PR labels (test:* pattern)
          elif [[ -n "${{ steps.labels.outputs.labels }}" ]]; then
            for label in $(echo "${{ steps.labels.outputs.labels }}" | tr ',' ' '); do
              if [[ "$label" == test:* && $EXTRA_COUNT -lt $MAX_EXTRA_TAGS ]]; then
                tag="@${label#test:}"
                tag="${tag#@}"  # Remove double @
                tag="@$tag"
                TAGS="$TAGS,$tag"
                EXTRA_COUNT=$((EXTRA_COUNT + 1))
                SOURCE="pr-labels"
                echo "🏷️ Label '$label' → $tag"
              fi
            done
          fi

          if [[ $EXTRA_COUNT -ge $MAX_EXTRA_TAGS ]]; then
            echo "⚠️ Tag limit reached ($MAX_EXTRA_TAGS max). Some tags were not added."
          fi

          # Priority 3: Auto-detected from PR changes (always additive, consolidated into ONE job)
          AUTO_TAGS="${{ steps.detect.outputs.auto_tags }}"
          if [[ -n "$AUTO_TAGS" ]]; then
            # Remove any auto-detected tags that already appear in manual/label TAGS
            # to prevent the same tests running in two separate matrix jobs
            EXISTING_TAGS=$(echo "$TAGS" | tr ',' '\n' | sort -u)
            FILTERED_AUTO=""
            for auto_tag in $AUTO_TAGS; do
              if echo "$EXISTING_TAGS" | grep -qx "$auto_tag"; then
                echo "⏭️  Skipping $auto_tag from auto-detected (already in manual/label tags)"
              else
                FILTERED_AUTO="$FILTERED_AUTO $auto_tag"
              fi
            done
            FILTERED_AUTO=$(echo "$FILTERED_AUTO" | xargs)

            if [[ -n "$FILTERED_AUTO" ]]; then
              # Consolidate remaining auto-detected tags into a single matrix entry
              # Cypress grep treats space-separated tags as OR, so one job covers all areas
              AUTO_DETECTED_ENTRY="$FILTERED_AUTO"
              if [[ "$SOURCE" == "default" ]]; then
                SOURCE="auto-detected"
              else
                SOURCE="$SOURCE+auto-detected"
              fi
              echo "🤖 Auto-detected tags (consolidated into 1 job): $AUTO_DETECTED_ENTRY"
            else
              echo "ℹ️  All auto-detected tags already covered by manual/label tags"
            fi
          fi

          # Convert to JSON matrix (deduplicated)
          MATRIX=$(echo "$TAGS" | tr ',' '\n' | sort -u | grep -v '^$' | \
            sed 's/^[^@]/@&/' | jq -Rc '[., inputs] | unique' | jq -sc 'add | unique')

          # Append the consolidated auto-detected entry as a single matrix item
          if [[ -n "$AUTO_DETECTED_ENTRY" ]]; then
            MATRIX=$(echo "$MATRIX" | jq -c --arg entry "$AUTO_DETECTED_ENTRY" '. + [$entry] | unique')
          fi
          
          # Ensure compact JSON for GitHub Actions output
          MATRIX=$(echo "$MATRIX" | jq -c '.')
          
          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
          echo "source=$SOURCE" >> $GITHUB_OUTPUT
          echo "🧪 Final matrix: $MATRIX (source: $SOURCE)"

  # ---------------------------------------------------------------------------
  # E2E Tests - Run Cypress tests for each tag in parallel
  # ---------------------------------------------------------------------------
  e2e-tests:
    needs: [select-cluster, set-pending-status, get-test-tags]
    runs-on: self-hosted
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        tag: ${{ fromJson(needs.get-test-tags.outputs.matrix) }}
    env:
      CLUSTER_NAME: ${{ needs.select-cluster.outputs.cluster_name }}
    steps:
      - name: Check Disk Space
        run: |
          echo "📊 Checking available disk space..."
          DISK_USAGE=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
          DISK_AVAIL=$(df -h / | tail -1 | awk '{print $4}')
          
          echo "💾 Disk usage: ${DISK_USAGE}% (${DISK_AVAIL} available)"
          echo "DISK_USAGE=$DISK_USAGE" >> $GITHUB_ENV
          
          if [ "$DISK_USAGE" -ge 95 ]; then
            echo "❌ CRITICAL: Disk usage is ${DISK_USAGE}% - will attempt emergency cleanup"
            echo "EMERGENCY_CLEANUP=true" >> $GITHUB_ENV
          elif [ "$DISK_USAGE" -ge 90 ]; then
            echo "⚠️  HIGH: Disk usage is ${DISK_USAGE}% - will attempt aggressive cleanup"
            echo "EMERGENCY_CLEANUP=true" >> $GITHUB_ENV
          elif [ "$DISK_USAGE" -ge 85 ]; then
            echo "⚠️  WARNING: Disk usage is ${DISK_USAGE}% - cleanup recommended"
            echo "   The cleanup job will run after this workflow completes"
            echo "EMERGENCY_CLEANUP=false" >> $GITHUB_ENV
          else
            echo "✅ Disk space OK (${DISK_USAGE}% used)"
            echo "EMERGENCY_CLEANUP=false" >> $GITHUB_ENV
          fi

      - name: Emergency Cleanup (if disk space critical)
        if: env.EMERGENCY_CLEANUP == 'true'
        run: |
          echo "🚨 EMERGENCY CLEANUP - Disk usage: ${DISK_USAGE}%"
          
          RUNNER_USER=$(whoami)
          HOME_DIR=$(eval echo "~$RUNNER_USER")
          CURRENT_WORK_DIR="${{ github.workspace }}"
          
          # Determine how aggressive to be based on disk usage
          if [ "$DISK_USAGE" -ge 95 ]; then
            AGE_THRESHOLD=1  # CRITICAL: Clean anything >1 day old
            echo "⚠️  CRITICAL MODE: Cleaning files >1 day old"
          else
            AGE_THRESHOLD=7  # Normal: Clean anything >7 days old
            echo "⚠️  AGGRESSIVE MODE: Cleaning files >7 days old"
          fi
          
          echo ""
          if [ "$DISK_USAGE" -ge 95 ]; then
            echo "🛡️  PARALLEL-SAFE PROTECTIONS (FAST MODE - disk critically full):"
            echo "   ✓ Current workspace (this job)"
            echo "   ✓ Active GitHub Actions Runner.Worker process directories"
            echo "   ⚡ Skipping slow checks (lsof, find) for speed"
          else
            echo "🛡️  PARALLEL-SAFE PROTECTIONS (THOROUGH MODE):"
            echo "   ✓ Current workspace (this job)"
            echo "   ✓ Active GitHub Actions Runner.Worker processes"
            echo "   ✓ Directories with open files (lsof with 5s timeout)"
            echo "   ✓ Directories accessed in last 10 minutes"
          fi
          echo ""
          
          # Get list of ALL active work directories from currently running GitHub Actions jobs
          # This is the safest way to avoid deleting directories from parallel PRs
          echo "🔍 Detecting active work directories from parallel jobs (with timeout)..."
          ACTIVE_WORK_DIRS=()
          ACTIVE_JOBS=0
          
          # Use faster method: check for active processes, then only protect their workspace
          ACTIVE_PIDS=$(pgrep -f "Runner.Worker" -u "$RUNNER_USER" 2>/dev/null || true)
          if [ -n "$ACTIVE_PIDS" ]; then
            echo "   Found active Runner.Worker processes: $ACTIVE_PIDS"
            # Get working directories of active processes using lsof (much faster than find)
            for pid in $ACTIVE_PIDS; do
              ACTIVE_JOBS=$((ACTIVE_JOBS + 1))
              # Get the CWD of this process
              if [ -L "/proc/$pid/cwd" ]; then
                WORK_CWD=$(readlink "/proc/$pid/cwd" 2>/dev/null || true)
                if [[ "$WORK_CWD" == *"odh-dashboard"* ]]; then
                  # Extract the odh-dashboard directory path
                  WORK_DIR=$(echo "$WORK_CWD" | sed 's|/odh-dashboard/.*|/odh-dashboard|')
                  ACTIVE_WORK_DIRS+=("$WORK_DIR")
                  echo "   🛡️  Protected: $WORK_DIR (PID $pid)"
                fi
              fi
            done
          fi
          echo "   Found $ACTIVE_JOBS active runner(s) with ${#ACTIVE_WORK_DIRS[@]} protected work directory(ies)"
          
          # Helper function to check if directory is in use by active runner
          is_directory_in_use() {
            local dir="$1"
            
            # 1. Skip current workspace (absolute must)
            if [[ "$dir" == "$CURRENT_WORK_DIR"* ]]; then
              return 0  # In use (current job)
            fi
            
            # 2. Check if directory is in the active work dirs list (FAST)
            for active_dir in "${ACTIVE_WORK_DIRS[@]}"; do
              if [[ "$dir" == "$active_dir"* ]]; then
                return 0  # In use (active job)
              fi
            done
            
            # 3. In CRITICAL mode (disk ≥95%), skip slow checks - rely on active work dirs only
            if [ "$DISK_USAGE" -ge 95 ]; then
              return 1  # Not in active list, safe to delete (fast path)
            fi
            
            # 4. Normal mode: Do thorough checks
            # Check for ANY processes using this directory (can be slow)
            if timeout 5 lsof +D "$dir" 2>/dev/null | grep -q .; then
              return 0  # In use (has open files)
            fi
            
            # 5. Check if directory was accessed very recently (last 10 minutes only)
            if find "$dir" -maxdepth 0 -amin -10 2>/dev/null | grep -q .; then
              return 0  # In use (very recent activity)
            fi
            
            return 1  # Not in use (safe to delete)
          }
          
          echo ""
          echo "🗑️  Step 1: Cleaning Go upstream builds (age: >$AGE_THRESHOLD days)..."
          UPSTREAM_CLEANED=0
          find "$HOME_DIR"/actions-runner*/_work -type d -path "*/packages/*/upstream" -mtime +$AGE_THRESHOLD 2>/dev/null | while read upstream_dir; do
            # Extract work_dir by going up to odh-dashboard parent
            work_dir=$(echo "$upstream_dir" | sed 's|/odh-dashboard/.*|/odh-dashboard|')
            if [ -n "$work_dir" ] && ! is_directory_in_use "$work_dir"; then
              rm -rf "$upstream_dir" 2>/dev/null && echo "   ✅ Cleaned: $upstream_dir" && UPSTREAM_CLEANED=$((UPSTREAM_CLEANED + 1)) || true
            fi
          done
          
          echo ""
          echo "🗑️  Step 2: Cleaning old work directories (age: >$AGE_THRESHOLD days, with multi-layer safety)..."
          CLEANED_COUNT=0
          SKIPPED_COUNT=0
          find "$HOME_DIR"/actions-runner*/_work -maxdepth 1 -name "odh-dashboard" -type d -mtime +$AGE_THRESHOLD 2>/dev/null | while read work_dir; do
            if [ -d "$work_dir" ]; then
              if is_directory_in_use "$work_dir"; then
                echo "   ⏭️  Protected (in use): $work_dir"
                SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
              else
                SIZE_BEFORE=$(du -sh "$work_dir" 2>/dev/null | awk '{print $1}')
                if rm -rf "$work_dir" 2>/dev/null; then
                  echo "   ✅ Cleaned $SIZE_BEFORE: $work_dir"
                  CLEANED_COUNT=$((CLEANED_COUNT + 1))
                fi
              fi
            fi
          done
          echo "   📊 Cleaned: $CLEANED_COUNT, Protected: $SKIPPED_COUNT"
          
          echo ""
          echo "🗑️  Step 3: Cleaning Cypress artifacts (age: >$AGE_THRESHOLD days)..."
          SCREENSHOTS_CLEANED=$(find "$HOME_DIR"/actions-runner*/_work -path "*/cypress/results/screenshots/*" -mtime +$AGE_THRESHOLD -delete -print 2>/dev/null | wc -l)
          VIDEOS_CLEANED=$(find "$HOME_DIR"/actions-runner*/_work -path "*/cypress/results/videos/*" -mtime +$AGE_THRESHOLD -delete -print 2>/dev/null | wc -l)
          echo "   ✅ Cleaned $SCREENSHOTS_CLEANED screenshots, $VIDEOS_CLEANED videos"
          
          echo ""
          echo "🗑️  Step 4: Cleaning runner logs (age: >$AGE_THRESHOLD days)..."
          LOGS_CLEANED=$(find "$HOME_DIR"/actions-runner*/_diag -name "*.log" -mtime +$AGE_THRESHOLD -delete -print 2>/dev/null | wc -l)
          echo "   ✅ Cleaned $LOGS_CLEANED log files"
          
          echo ""
          echo "🗑️  Step 5: Cleaning node_modules in old work dirs (age: >$AGE_THRESHOLD days)..."
          find "$HOME_DIR"/actions-runner*/_work -type d -name "node_modules" -mtime +$AGE_THRESHOLD 2>/dev/null | while read nm_dir; do
            # Extract work_dir by going up to odh-dashboard parent
            work_dir=$(echo "$nm_dir" | sed 's|/odh-dashboard/.*|/odh-dashboard|')
            if [ -n "$work_dir" ] && ! is_directory_in_use "$work_dir"; then
              SIZE_BEFORE=$(du -sh "$nm_dir" 2>/dev/null | awk '{print $1}')
              rm -rf "$nm_dir" 2>/dev/null && echo "   ✅ Cleaned $SIZE_BEFORE node_modules: $nm_dir" || true
            fi
          done
          
          echo ""
          echo "🗑️  Step 6: Cleaning .turbo cache in old work dirs (age: >$AGE_THRESHOLD days)..."
          find "$HOME_DIR"/actions-runner*/_work -type d -name ".turbo" -mtime +$AGE_THRESHOLD 2>/dev/null | while read turbo_dir; do
            # Extract work_dir by going up to odh-dashboard parent
            work_dir=$(echo "$turbo_dir" | sed 's|/odh-dashboard/.*|/odh-dashboard|')
            if [ -n "$work_dir" ] && ! is_directory_in_use "$work_dir"; then
              SIZE_BEFORE=$(du -sh "$turbo_dir" 2>/dev/null | awk '{print $1}')
              rm -rf "$turbo_dir" 2>/dev/null && echo "   ✅ Cleaned $SIZE_BEFORE .turbo: $turbo_dir" || true
            fi
          done
          
          echo ""
          echo "📊 Disk usage after emergency cleanup:"
          DISK_USAGE_AFTER=$(df / | tail -1 | awk '{print $5}' | sed 's/%//')
          DISK_AVAIL_AFTER=$(df -h / | tail -1 | awk '{print $4}')
          echo "💾 Disk usage: ${DISK_USAGE_AFTER}% (${DISK_AVAIL_AFTER} available)"
          
          FREED=$((DISK_USAGE - DISK_USAGE_AFTER))
          if [ "$FREED" -gt 0 ]; then
            echo "✅ Freed: ${FREED}% disk space"
          else
            echo "⚠️  Freed: 0% disk space (no files met age threshold)"
          fi
          
          # Show what's taking up space
          echo ""
          echo "📊 Top disk usage on runner:"
          du -sh "$HOME_DIR"/actions-runner*/_work/* 2>/dev/null | sort -rh | head -5 || true
          
          # Decide whether to fail or continue
          if [ "$DISK_USAGE_AFTER" -ge 95 ]; then
            if [ "$FREED" -gt 0 ]; then
              echo "⚠️  WARNING: Still at ${DISK_USAGE_AFTER}% after cleanup, but freed ${FREED}%"
              echo "   Attempting to proceed - job may fail if more space is needed"
            else
              echo "❌ CRITICAL: Still at ${DISK_USAGE_AFTER}% after cleanup and freed 0%"
              echo "   All work directories are either:"
              echo "   - Currently in use by active jobs"
              echo "   - Created within the last $AGE_THRESHOLD day(s)"
              echo ""
              echo "🔍 Diagnosis - Active work directories:"
              find "$HOME_DIR"/actions-runner*/_work -maxdepth 1 -name "odh-dashboard" -type d 2>/dev/null | while read work_dir; do
                MTIME=$(stat -f %m "$work_dir" 2>/dev/null || stat -c %Y "$work_dir" 2>/dev/null || echo "0")
                AGE_DAYS=$(( ($(date +%s) - MTIME) / 86400 ))
                SIZE=$(du -sh "$work_dir" 2>/dev/null | awk '{print $1}')
                echo "   - $work_dir: $SIZE, age: ${AGE_DAYS} days"
              done
              echo ""
              echo "   Manual intervention required on runner $(hostname)"
              exit 1
            fi
          elif [ "$DISK_USAGE_AFTER" -ge 90 ]; then
            echo "⚠️  WARNING: Still at ${DISK_USAGE_AFTER}% after cleanup"
            echo "   Job will proceed but may fail due to space"
          else
            echo "✅ Cleanup successful - proceeding with tests"
          fi

      - name: Calculate unique port for this workflow run
        run: |
          # Dynamic port allocation for parallel execution
          BASE_PORT=$((4000 + (${{ github.run_id }} % 1000) * 5))

          # Add matrix offset to separate concurrent jobs within same PR
          if [[ "${{ matrix.tag }}" == *"set-1"* ]]; then
            MATRIX_OFFSET=0
          elif [[ "${{ matrix.tag }}" == *"set-2"* ]]; then
            MATRIX_OFFSET=1
          else
            MATRIX_OFFSET=2
          fi

          WEBPACK_PORT=$((BASE_PORT + MATRIX_OFFSET))
          PORT_INFO_DIR="/tmp/gha-ports"
          mkdir -p "$PORT_INFO_DIR"
          
          echo "📍 Calculated port ${WEBPACK_PORT} for ${{ matrix.tag }} (run_id: ${{ github.run_id }})"
          
          # SAFE port conflict resolution - only clean orphaned processes
          if lsof -ti:${WEBPACK_PORT} > /dev/null 2>&1; then
            echo "⚠️  Port ${WEBPACK_PORT} is currently in use - checking ownership..."
            
            PORT_OWNER_FILE="$PORT_INFO_DIR/port-${WEBPACK_PORT}.run_id"
            if [ -f "$PORT_OWNER_FILE" ]; then
              OWNER_RUN_ID=$(cat "$PORT_OWNER_FILE")
              FILE_AGE=$(($(date +%s) - $(stat -f %m "$PORT_OWNER_FILE" 2>/dev/null || stat -c %Y "$PORT_OWNER_FILE" 2>/dev/null)))
              
              # Only kill if the owning run is OLD (>30 minutes = likely completed/stuck)
              if [ "$FILE_AGE" -gt 1800 ]; then
                echo "🧹 Port owned by old run_id $OWNER_RUN_ID (${FILE_AGE}s old) - cleaning up..."
                PORT_PID=$(lsof -ti:${WEBPACK_PORT} 2>/dev/null | head -1)
                if [ -n "$PORT_PID" ]; then
                  pkill -P "$PORT_PID" 2>/dev/null || true
                  kill -9 "$PORT_PID" 2>/dev/null || true
                  sleep 2
                  if lsof -ti:${WEBPACK_PORT} > /dev/null 2>&1; then
                    echo "❌ Failed to clean orphaned process - port still in use"
                    echo "   This may require manual intervention on the runner"
                    exit 1
                  else
                    echo "✅ Successfully cleaned orphaned process"
                  fi
                fi
              else
                echo "⚠️  Port owned by recent run_id $OWNER_RUN_ID (${FILE_AGE}s old)"
                echo "   This is likely an active parallel PR test - DO NOT KILL"
                echo "❌ Port conflict detected - please retry workflow in a few minutes"
                exit 1
              fi
            else
              echo "⚠️  Port in use but no owner info found - checking process age..."
              PORT_PID=$(lsof -ti:${WEBPACK_PORT} 2>/dev/null | head -1)
              if [ -n "$PORT_PID" ]; then
                # Check process start time (macOS: -o etime, Linux: -o etimes)
                PROCESS_AGE=$(ps -o etimes= -p "$PORT_PID" 2>/dev/null || echo "unknown")
                if [ "$PROCESS_AGE" != "unknown" ] && [ "$PROCESS_AGE" -gt 1800 ]; then
                  echo "🧹 Orphaned process (${PROCESS_AGE}s old) - cleaning up..."
                  pkill -P "$PORT_PID" 2>/dev/null || true
                  kill -9 "$PORT_PID" 2>/dev/null || true
                  sleep 2
                else
                  echo "❌ Port in use by recent process - may be parallel PR test"
                  echo "   Please retry workflow in a few minutes"
                  exit 1
                fi
              fi
            fi
          else
            echo "✅ Port ${WEBPACK_PORT} is available"
          fi

          # Store port info with run_id for cleanup tracking
          echo "${{ github.run_id }}" > "$PORT_INFO_DIR/port-${WEBPACK_PORT}.run_id"

          echo "WEBPACK_PORT=$WEBPACK_PORT" >> $GITHUB_ENV
          echo "PORT_INFO_FILE=$PORT_INFO_DIR/port-${WEBPACK_PORT}.run_id" >> $GITHUB_ENV

      - name: Cleanup old test artifacts
        continue-on-error: true
        run: |
          echo "🧹 Cleaning up old test artifacts (>2 days)..."
          
          # Clean old Cypress results/screenshots/videos from workspace (>2 days old)
          find ${{ github.workspace }}/packages/cypress/results -type f -mtime +2 -delete 2>/dev/null || true
          find ${{ github.workspace }}/packages/cypress/screenshots -type f -mtime +2 -delete 2>/dev/null || true
          find ${{ github.workspace }}/packages/cypress/videos -type f -mtime +2 -delete 2>/dev/null || true
          
          # Clean old webpack logs (>2 days old)
          find /tmp -name "webpack_*.log" -type f -mtime +2 -delete 2>/dev/null || true
          
          # Note: ~/.cache/Cypress is managed by actions/cache and should not be cleaned here
          # to avoid removing the Cypress binary on shared self-hosted runners
          
          # Clean old temporary yaml files (>2 days old)
          find /tmp -name "cypress-yaml-*.yaml" -type f -mtime +2 -delete 2>/dev/null || true
          
          # Clean empty directories
          find ${{ github.workspace }}/packages/cypress/results -type d -empty -delete 2>/dev/null || true
          find ${{ github.workspace }}/packages/cypress/screenshots -type d -empty -delete 2>/dev/null || true
          find ${{ github.workspace }}/packages/cypress/videos -type d -empty -delete 2>/dev/null || true
          
          echo "✅ Cleanup complete (non-critical, continued on any errors)"

      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.workflow_run.head_sha || github.sha }}

      - name: Restore npm dependencies cache
        uses: actions/cache/restore@v4
        id: npm-cache
        with:
          path: |
            ~/.cache/Cypress
            **/node_modules
          key: ${{ runner.os }}-${{ env.NODE_VERSION }}-all-modules-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-${{ env.NODE_VERSION }}-all-modules-

      - name: Setup Node.js ${{ env.NODE_VERSION }}
        if: steps.npm-cache.outputs.cache-hit != 'true'
        uses: actions/setup-node@v4.3.0
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Install dependencies
        if: steps.npm-cache.outputs.cache-hit != 'true'
        run: npm ci

      - name: Restore turbo build artifacts cache
        uses: actions/cache/restore@v4
        with:
          path: ${{ github.workspace }}/.turbo
          key: ${{ runner.os }}-${{ env.NODE_VERSION }}-turbo-${{ github.sha }}-e2e
          restore-keys: |
            ${{ runner.os }}-${{ env.NODE_VERSION }}-turbo-

      - name: Restore OpenShift CLI tarball cache
        uses: actions/cache/restore@v4
        id: oc-cache
        with:
          path: ${{ runner.temp }}/oc.tar.gz
          key: ${{ runner.os }}-oc-tarball-${{ env.OC_VERSION || '4.15.0' }}

      - name: Download test configuration
        run: |
          echo "🔧 Downloading test configuration from GitLab..."
          curl -fk -H "Authorization: Bearer ${{ secrets.GITLAB_TOKEN }}" \
                        "${{ secrets.GITLAB_TEST_VARS_URL }}" \
            -o ${{ github.workspace }}/packages/cypress/test-variables.yml
          echo "✅ Downloaded test configuration"

      - name: Login to OpenShift cluster
        env:
          OC_SERVER_PRIMARY: ${{ secrets.OC_SERVER_PRIMARY }}
          OC_SERVER_SECONDARY: ${{ secrets.OC_SERVER }}
        run: |
          TEST_VARS_FILE="${{ github.workspace }}/packages/cypress/test-variables.yml"

          # Extract credentials based on test type
          if [[ "${{ matrix.tag }}" == "@NonAdmin" ]]; then
            echo "🔑 Using non-admin credentials (TEST_USER_3) for @NonAdmin tests"
            OC_USERNAME=$(grep -A 10 "^TEST_USER_3:" "$TEST_VARS_FILE" | grep "USERNAME:" | head -1 | sed 's/.*USERNAME: //' | tr -d ' ')
            OC_PASSWORD=$(grep -A 10 "^TEST_USER_3:" "$TEST_VARS_FILE" | grep "PASSWORD:" | head -1 | sed 's/.*PASSWORD: //' | tr -d ' ')
          else
            OC_USERNAME=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "USERNAME:" | head -1 | sed 's/.*USERNAME: //' | tr -d ' ')
            OC_PASSWORD=$(grep -A 10 "^OCP_ADMIN_USER:" "$TEST_VARS_FILE" | grep "PASSWORD:" | head -1 | sed 's/.*PASSWORD: //' | tr -d ' ')
          fi
          echo "::add-mask::$OC_PASSWORD"
          echo "::add-mask::$OC_USERNAME"

          # Look up server URL based on selected cluster (avoids GitHub secret masking in outputs)
          if [ "$CLUSTER_NAME" = "dash-e2e-int" ]; then
            CLUSTER_URL="$OC_SERVER_PRIMARY"
          elif [ "$CLUSTER_NAME" = "dash-e2e" ]; then
            CLUSTER_URL="$OC_SERVER_SECONDARY"
          else
            echo "❌ Unknown or empty CLUSTER_NAME: '$CLUSTER_NAME'" >&2
            echo "Expected 'dash-e2e-int' or 'dash-e2e'" >&2
            exit 1
          fi

          if [ -z "$CLUSTER_URL" ]; then
            echo "❌ CLUSTER_URL is empty for cluster '$CLUSTER_NAME'" >&2
            echo "Check that OC_SERVER_PRIMARY/OC_SERVER secrets are configured" >&2
            exit 1
          fi

          echo "Logging in to OpenShift cluster ($CLUSTER_NAME)..."
          oc login -u "$OC_USERNAME" -p "$OC_PASSWORD" --server="$CLUSTER_URL" --insecure-skip-tls-verify > /dev/null 2>&1

          if [ $? -eq 0 ]; then
            echo "✅ Successfully logged in to $CLUSTER_NAME"
          else
            echo "❌ Failed to login to OpenShift cluster"
            exit 1
          fi

          echo "KUBECONFIG=$HOME/.kube/config" >> $GITHUB_ENV

      - name: Override namespace values
        env:
          DASHBOARD_URL_PRIMARY: ${{ secrets.ODH_DASHBOARD_URL_PRIMARY }}
          DASHBOARD_URL_SECONDARY: ${{ secrets.ODH_DASHBOARD_URL }}
          ODH_NAMESPACES: ${{ secrets.ODH_NAMESPACES }}
        run: |
          TEST_VARS_FILE="${{ github.workspace }}/packages/cypress/test-variables.yml"

          # Look up dashboard URL based on selected cluster (secrets passed as step-level env for security)
          if [ "$CLUSTER_NAME" = "dash-e2e-int" ]; then
            DASHBOARD_URL="$DASHBOARD_URL_PRIMARY"
          elif [ "$CLUSTER_NAME" = "dash-e2e" ]; then
            DASHBOARD_URL="$DASHBOARD_URL_SECONDARY"
          else
            echo "❌ Unknown or empty CLUSTER_NAME: '$CLUSTER_NAME'" >&2
            echo "Expected 'dash-e2e-int' or 'dash-e2e'" >&2
            exit 1
          fi

          if [ -z "$DASHBOARD_URL" ]; then
            echo "❌ DASHBOARD_URL is empty for cluster '$CLUSTER_NAME'" >&2
            echo "Check that ODH_DASHBOARD_URL_PRIMARY/ODH_DASHBOARD_URL secrets are configured" >&2
            exit 1
          fi

          # Mask dashboard URL to prevent exposure in logs
          echo "::add-mask::$DASHBOARD_URL"

          # Set dashboard URL for selected cluster
          sed -i "s|^ODH_DASHBOARD_URL:.*|ODH_DASHBOARD_URL: $DASHBOARD_URL|" "$TEST_VARS_FILE"

          # Export dashboard host (without protocol) for webpack ODH_DASHBOARD_HOST
          DASHBOARD_HOST=$(echo "$DASHBOARD_URL" | sed -E 's|https?://||' | sed 's|/.*||')
          echo "::add-mask::$DASHBOARD_HOST"
          echo "DASHBOARD_HOST=$DASHBOARD_HOST" >> $GITHUB_ENV

          if [ -z "$ODH_NAMESPACES" ]; then
            echo "⚠️ ODH_NAMESPACES secret not set, skipping namespace override"
            exit 0
          fi

          echo "::add-mask::$ODH_NAMESPACES"
          echo "📝 Overriding namespaces with ODH values..."

          IFS=',' read -r OPERATOR_NS APPLICATIONS_NS NOTEBOOKS_NS OPERATOR_NAME PROJECT_NAME <<< "$ODH_NAMESPACES"

          sed -i "s|^PRODUCT:.*|PRODUCT: ODH|" "$TEST_VARS_FILE"
          sed -i "s|^OPERATOR_NAMESPACE:.*|OPERATOR_NAMESPACE: $OPERATOR_NS|" "$TEST_VARS_FILE"
          sed -i "s|^APPLICATIONS_NAMESPACE:.*|APPLICATIONS_NAMESPACE: $APPLICATIONS_NS|" "$TEST_VARS_FILE"
          sed -i "s|^MONITORING_NAMESPACE:.*|MONITORING_NAMESPACE: $APPLICATIONS_NS|" "$TEST_VARS_FILE"
          sed -i "s|^NOTEBOOKS_NAMESPACE:.*|NOTEBOOKS_NAMESPACE: $NOTEBOOKS_NS|" "$TEST_VARS_FILE"
          sed -i "s|^OPERATOR_NAME:.*|OPERATOR_NAME: $OPERATOR_NAME|" "$TEST_VARS_FILE"
          sed -i "s|^ODH_DASHBOARD_PROJECT_NAME:.*|ODH_DASHBOARD_PROJECT_NAME: $PROJECT_NAME|" "$TEST_VARS_FILE"

          echo "✅ Namespace configuration updated"

      - name: Set test configuration
        run: |
          echo "CY_TEST_CONFIG=${{ github.workspace }}/packages/cypress/test-variables.yml" >> $GITHUB_ENV

      - name: Start Cypress Server
        run: |
          echo "🧹 Cleaning up port ${WEBPACK_PORT}..."

          PORT_INFO_DIR="/tmp/gha-ports"
          PORT_INFO_FILE="$PORT_INFO_DIR/port-${WEBPACK_PORT}.run_id"
          CURRENT_RUN_ID="${{ github.run_id }}"

          # Check if port is in use
          if lsof -i:${WEBPACK_PORT} > /dev/null 2>&1; then
            # Check if there's a run_id file for this port
            if [ -f "$PORT_INFO_FILE" ]; then
              PORT_OWNER_RUN_ID=$(cat "$PORT_INFO_FILE")
              if [ "$PORT_OWNER_RUN_ID" != "$CURRENT_RUN_ID" ]; then
                echo "⚠️  Port ${WEBPACK_PORT} is owned by different run_id: $PORT_OWNER_RUN_ID"
                echo "⚠️  This port is in use by another workflow run - will not kill it"
                # Try to find an alternative port
                for alt_port in $(seq $((WEBPACK_PORT + 5)) $((WEBPACK_PORT + 50)) 5); do
                  if ! lsof -i:${alt_port} > /dev/null 2>&1; then
                    WEBPACK_PORT=$alt_port
                    PORT_INFO_FILE="$PORT_INFO_DIR/port-${WEBPACK_PORT}.run_id"
                    echo "✅ Found alternative port: ${WEBPACK_PORT}"
                    break
                  fi
                done
              else
                echo "✅ Port ${WEBPACK_PORT} is owned by this run - safe to clean up"
              fi
            else
              # No run_id file - check if process is from a recent GitHub Actions run
              PORT_PID=$(lsof -ti:${WEBPACK_PORT} 2>/dev/null | head -1)
              if [ -n "$PORT_PID" ]; then
                # Check if process is from a GitHub Actions workflow
                if ps -p "$PORT_PID" -o command= 2>/dev/null | grep -q "webpack.*serve\|node.*40[0-9][0-9]"; then
                  echo "⚠️  Port ${WEBPACK_PORT} in use by potential GHA process (PID: $PORT_PID)"
                  echo "⚠️  Being cautious - will not kill without run_id confirmation"
                  # Find alternative port
                  for alt_port in $(seq $((WEBPACK_PORT + 5)) $((WEBPACK_PORT + 50)) 5); do
                    if ! lsof -i:${alt_port} > /dev/null 2>&1; then
                      WEBPACK_PORT=$alt_port
                      PORT_INFO_FILE="$PORT_INFO_DIR/port-${WEBPACK_PORT}.run_id"
                      echo "✅ Found alternative port: ${WEBPACK_PORT}"
                      break
                    fi
                  done
                else
                  echo "⚠️  Port ${WEBPACK_PORT} in use by non-GHA process - cleaning up"
                  kill -9 "$PORT_PID" 2>/dev/null || true
                fi
              fi
            fi
          fi

          # Verify port is free with retry logic
          RETRY_COUNT=0
          while lsof -i:${WEBPACK_PORT} > /dev/null 2>&1; do
            RETRY_COUNT=$((RETRY_COUNT + 1))
            if [ $RETRY_COUNT -gt 10 ]; then
              echo "❌ Port ${WEBPACK_PORT} still in use after cleanup!"
              lsof -i:${WEBPACK_PORT}
              exit 1
            fi
            echo "⏳ Retrying cleanup... (attempt $RETRY_COUNT/10)"
            sleep 2
          done

          # Claim the port with our run_id
          mkdir -p "$PORT_INFO_DIR"
          echo "$CURRENT_RUN_ID" > "$PORT_INFO_FILE"
          echo "WEBPACK_PORT=$WEBPACK_PORT" >> $GITHUB_ENV
          echo "PORT_INFO_FILE=$PORT_INFO_FILE" >> $GITHUB_ENV

          echo "✅ Port ${WEBPACK_PORT} is free and claimed by run_id: $CURRENT_RUN_ID"
          echo "🚀 Starting webpack dev server on port ${WEBPACK_PORT} ($CLUSTER_NAME)..."

          # Start webpack with explicit dashboard host (ensures correct proxy target for all tests)
          cd frontend && env ODH_DASHBOARD_HOST=${DASHBOARD_HOST} ODH_PORT=${WEBPACK_PORT} npm run start:dev:ext > /tmp/webpack_${WEBPACK_PORT}.log 2>&1 &
          SERVER_PID=$!
          echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV
          echo "$SERVER_PID" > "$PORT_INFO_DIR/port-${WEBPACK_PORT}.pid"

          # Give server time to initialize
          sleep 20

          # Show filtered webpack status (hide sensitive cluster URLs)
          if [ -f /tmp/webpack_${WEBPACK_PORT}.log ]; then
            tail -20 /tmp/webpack_${WEBPACK_PORT}.log | \
              grep -v "Dashboard host:" | \
              grep -v "Proxy created:" | \
              grep -v "Logged in as user:" | \
              grep -v "Using project:" || true
          fi

      - name: Install make for BFF builds
        if: needs.get-test-tags.outputs.bff_packages != '[]'
        run: |
          if ! command -v make &> /dev/null; then
            echo "📦 Installing make..."
            sudo dnf install -y make 2>/dev/null || sudo yum install -y make 2>/dev/null || sudo apt-get install -y make 2>/dev/null
          else
            echo "✅ make already available: $(make --version | head -1)"
          fi

      - name: Setup Go for BFF builds
        if: needs.get-test-tags.outputs.bff_packages != '[]'
        uses: actions/setup-go@v5
        with:
          go-version: '1.24'
          cache: false

      - name: Start BFF Services
        if: needs.get-test-tags.outputs.bff_packages != '[]'
        env:
          BFF_PACKAGES: ${{ needs.get-test-tags.outputs.bff_packages }}
        run: |
          echo "🚀 Starting BFF services for changed packages..."

          BFF_INFO_DIR="/tmp/gha-bff/${{ github.run_id }}"
          mkdir -p "$BFF_INFO_DIR"

          # Allowlists to prevent command injection from PR code (CWE-94)
          ALLOWED_COMMANDS=("make dev-bff-e2e-mock" "make dev-bff-e2e-cluster" "make -C upstream dev-bff-e2e-mock" "make -C upstream dev-bff-e2e-cluster")
          ALLOWED_DIRS=("automl" "autorag" "eval-hub" "gen-ai" "maas" "mlflow" "model-registry")

          # Parse BFF packages JSON and start each one
          echo "$BFF_PACKAGES" | jq -c '.[]' | while read -r bff_config; do
            BFF_NAME=$(echo "$bff_config" | jq -r '.name')
            BFF_DIR=$(echo "$bff_config" | jq -r '.dir')
            BFF_PORT=$(echo "$bff_config" | jq -r '.port')
            BFF_HEALTH=$(echo "$bff_config" | jq -r '.healthEndpoint')
            BFF_CMD=$(echo "$bff_config" | jq -r '.startCommand')

            echo ""
            echo "📦 Starting BFF for $BFF_NAME on port $BFF_PORT..."

            # Validate BFF_CMD against allowlist (prevent command injection)
            CMD_VALID=false
            for allowed in "${ALLOWED_COMMANDS[@]}"; do
              if [ "$BFF_CMD" = "$allowed" ]; then
                CMD_VALID=true
                break
              fi
            done
            if [ "$CMD_VALID" = "false" ]; then
              echo "  ❌ Rejected untrusted startCommand: '$BFF_CMD'"
              echo "  Allowed commands: ${ALLOWED_COMMANDS[*]}"
              exit 1
            fi

            # Validate BFF_DIR against allowlist (prevent path traversal)
            DIR_VALID=false
            for allowed in "${ALLOWED_DIRS[@]}"; do
              if [ "$BFF_DIR" = "$allowed" ]; then
                DIR_VALID=true
                break
              fi
            done
            if [ "$DIR_VALID" = "false" ]; then
              echo "  ❌ Rejected untrusted package directory: '$BFF_DIR'"
              echo "  Allowed directories: ${ALLOWED_DIRS[*]}"
              exit 1
            fi

            # Validate port is numeric and health endpoint is a safe path
            if ! echo "$BFF_PORT" | grep -qE '^[0-9]+$'; then
              echo "  ❌ Invalid BFF port: '$BFF_PORT' (must be numeric)"
              exit 1
            fi
            if ! echo "$BFF_HEALTH" | grep -qE '^/[a-zA-Z0-9/_-]+$'; then
              echo "  ❌ Invalid health endpoint: '$BFF_HEALTH' (must be a simple path)"
              exit 1
            fi

            # Reuse if already healthy on this port (avoid parallel startup race)
            if curl -sf "http://localhost:$BFF_PORT$BFF_HEALTH" > /dev/null 2>&1; then
              echo "  ♻️ BFF $BFF_NAME already healthy on $BFF_PORT, reusing existing process"
              continue
            fi

            # Fail fast if port is occupied but unhealthy
            if lsof -ti:"$BFF_PORT" > /dev/null 2>&1; then
              echo "  ❌ Port $BFF_PORT already in use and health check failed"
              lsof -i:"$BFF_PORT" || true
              exit 1
            fi

            # Start BFF in background with configurable port
            cd "packages/$BFF_DIR"
            E2E_BFF_PORT=$BFF_PORT $BFF_CMD > "/tmp/bff_${BFF_DIR}.log" 2>&1 &
            BFF_PID=$!
            cd - > /dev/null

            # Save PID for cleanup
            echo "$BFF_PID" >> "$BFF_INFO_DIR/pids.txt"
            echo "$BFF_PORT" >> "$BFF_INFO_DIR/ports.txt"

            echo "  📍 PID: $BFF_PID"

            # Wait for health check (max 60 seconds)
            echo "  ⏳ Waiting for health at localhost:$BFF_PORT$BFF_HEALTH..."
            for i in {1..30}; do
              if curl -sf "http://localhost:$BFF_PORT$BFF_HEALTH" > /dev/null 2>&1; then
                echo "  ✅ BFF $BFF_NAME is healthy!"
                break
              fi
              if [ $i -eq 30 ]; then
                echo "  ❌ BFF $BFF_NAME failed health check after 60 seconds"
                echo "  📋 BFF logs:"
                tail -50 "/tmp/bff_${BFF_DIR}.log" || true
                exit 1
              fi
              sleep 2
            done
          done

          echo ""
          echo "✅ All BFF services started successfully"

      - name: Wait for Server Ready
        run: |
          echo "⏳ Waiting for localhost:${WEBPACK_PORT} to be ready..."
          npx wait-on http://localhost:${WEBPACK_PORT} --timeout 120000

          # Verify the application loads with dashboard content
          for i in {1..10}; do
            if curl -s -f http://localhost:${WEBPACK_PORT}/ | grep -q "Data Science Projects\|ODH\|Open Data Hub\|Dashboard"; then
              echo "✅ Server is ready and application is loaded!"
              break
            fi
            
            if [ $i -lt 10 ]; then
              echo "⏳ Waiting for application to load... (attempt $i/10)"
              sleep 8
            else
              echo "❌ Application failed to load properly after 10 attempts"
              exit 1
            fi
          done

      - name: Run E2E Tests
        env:
          OC_SERVER_PRIMARY: ${{ secrets.OC_SERVER_PRIMARY }}
          OC_SERVER_SECONDARY: ${{ secrets.OC_SERVER }}
        run: |
          cd frontend

          echo "🧪 Running E2E tests for ${{ matrix.tag }}..."
          echo "🚀 Running tests against live dashboard on port ${WEBPACK_PORT}"
          echo "📌 Tag source: ${{ needs.get-test-tags.outputs.source }}"

          export CY_RESULTS_DIR="${{ github.workspace }}/packages/cypress/results/${{ matrix.tag }}"
          mkdir -p "$CY_RESULTS_DIR"

          # Determine OC_SERVER based on cluster (for oc user switching in tests)
          if [ "$CLUSTER_NAME" = "dash-e2e-int" ]; then
            OC_SERVER="$OC_SERVER_PRIMARY"
          elif [ "$CLUSTER_NAME" = "dash-e2e" ]; then
            OC_SERVER="$OC_SERVER_SECONDARY"
          else
            echo "⚠️ Unknown cluster: $CLUSTER_NAME, defaulting to OC_SERVER_PRIMARY"
            OC_SERVER="$OC_SERVER_PRIMARY"
          fi

          # Set IS_NON_ADMIN_RUN flag for non-admin tests to skip admin-only setup hooks
          EXTRA_CYPRESS_ENV="OC_SERVER=${OC_SERVER},"
          if [[ "${{ matrix.tag }}" == "@NonAdmin" ]]; then
            EXTRA_CYPRESS_ENV="${EXTRA_CYPRESS_ENV}IS_NON_ADMIN_RUN=true,"
            echo "🔐 Running in non-admin mode - admin setup hooks will be skipped"
          fi

          BASE_URL=http://localhost:${WEBPACK_PORT} npm run cypress:run:chrome -- \
            --env ${EXTRA_CYPRESS_ENV}skipTags="@Bug @Maintain @NonConcurrent",grepTags="${{ matrix.tag }}",grepFilterSpecs=true \
            --config video=true,screenshotsFolder="$CY_RESULTS_DIR/screenshots",videosFolder="$CY_RESULTS_DIR/videos"

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-results-${{ matrix.tag }}
          path: |
            packages/cypress/results/
            packages/cypress/videos/
            packages/cypress/screenshots/
          retention-days: 7

      - name: Log test completion
        if: always()
        run: |
          echo "🏁 E2E Test completed!"
          echo "Status: ${{ job.status }}"
          echo "Test Tag: ${{ matrix.tag }}"
          echo "Cluster: $CLUSTER_NAME"
          echo "Run ID: ${{ github.run_id }}"

  # ---------------------------------------------------------------------------
  # Final Status - Update PR with test results
  # ---------------------------------------------------------------------------
  set-final-status:
    needs: [select-cluster, e2e-tests]
    if: >-
      always() && 
      (github.event_name == 'workflow_dispatch' || 
       (github.event.workflow_run.event == 'pull_request' && 
        github.event.workflow_run.conclusion == 'success'))
    runs-on: ubuntu-latest
    steps:
      - name: Set final status
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          E2E_RESULT="${{ needs.e2e-tests.result }}"
          CLUSTER_RESULT="${{ needs.select-cluster.result }}"
          CLUSTER="${{ needs.select-cluster.outputs.cluster_name }}"
          
          echo "📊 Job results: select-cluster=$CLUSTER_RESULT, e2e-tests=$E2E_RESULT"
          
          # Handle cluster selection failure first
          if [[ "$CLUSTER_RESULT" == "failure" ]]; then
            STATE="failure"
            DESC="Cluster health check failed - no healthy cluster available"
          elif [[ "$E2E_RESULT" == "success" ]]; then
            STATE="success"
            DESC="All tests passed on $CLUSTER"
          elif [[ "$E2E_RESULT" == "cancelled" ]]; then
            STATE="error"
            DESC="Tests cancelled"
          elif [[ "$E2E_RESULT" == "skipped" && "$CLUSTER_RESULT" == "skipped" ]]; then
            # Both skipped means test.yml failed - don't post status
            echo "Both jobs skipped (test.yml likely failed) - not posting status"
            exit 0
          elif [[ "$E2E_RESULT" == "skipped" ]]; then
            STATE="failure"
            DESC="Tests skipped due to upstream failure"
          else
            STATE="failure"
            DESC="Tests failed on ${CLUSTER:-unknown cluster}"
          fi
          
          echo "📝 Posting status: state=$STATE, description=$DESC"
          
          gh api repos/${{ github.repository }}/statuses/${{ github.event.workflow_run.head_sha || github.sha }} \
            -f state="$STATE" \
            -f context="Cypress E2E Tests" \
            -f description="$DESC" \
            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  # ---------------------------------------------------------------------------
  # Cleanup - Stop all servers started by this workflow run
  # ---------------------------------------------------------------------------
  cleanup-server:
    needs: [e2e-tests]
    runs-on: self-hosted
    if: ${{ always() && (github.event_name == 'workflow_dispatch' || (github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success')) }}
    steps:
      - name: Stop BFF Services
        run: |
          echo "🛑 Stopping BFF services for run_id: ${{ github.run_id }}..."

          BFF_INFO_DIR="/tmp/gha-bff/${{ github.run_id }}"
          BFF_KILLED_COUNT=0

          # Check if this workflow started any BFFs
          if [ -d "$BFF_INFO_DIR" ]; then
            # Kill all BFF processes
            if [ -f "$BFF_INFO_DIR/pids.txt" ]; then
              while read -r pid; do
                if ps -p "$pid" > /dev/null 2>&1; then
                  echo "  🛑 Killing BFF process $pid"
                  pkill -P "$pid" 2>/dev/null || true
                  kill "$pid" 2>/dev/null || true
                  BFF_KILLED_COUNT=$((BFF_KILLED_COUNT + 1))
                fi
              done < "$BFF_INFO_DIR/pids.txt"
            fi

            # Clean up BFF ports
            if [ -f "$BFF_INFO_DIR/ports.txt" ]; then
              while read -r port; do
                BFF_PID=$(lsof -ti:${port} 2>/dev/null | head -1)
                if [ -n "$BFF_PID" ]; then
                  echo "  🛑 Killing process on BFF port $port (PID: $BFF_PID)"
                  pkill -P "$BFF_PID" 2>/dev/null || true
                  kill "$BFF_PID" 2>/dev/null || true
                fi
              done < "$BFF_INFO_DIR/ports.txt"
            fi

            # Clean up info files
            rm -rf "$BFF_INFO_DIR"
          fi

          if [ $BFF_KILLED_COUNT -eq 0 ]; then
            echo "✅ No BFF processes found for run_id: ${{ github.run_id }}"
          else
            echo "✅ Cleaned up $BFF_KILLED_COUNT BFF process(es) for run_id: ${{ github.run_id }}"
          fi

      - name: Stop Cypress Servers
        run: |
          echo "🛑 Stopping webpack dev server for run_id: ${{ github.run_id }}..."

          PORT_INFO_DIR="/tmp/gha-ports"
          CURRENT_RUN_ID="${{ github.run_id }}"
          KILLED_COUNT=0

          # Find all port files owned by this run_id
          if [ -d "$PORT_INFO_DIR" ]; then
            for port_file in "$PORT_INFO_DIR"/port-*.run_id; do
              if [ -f "$port_file" ]; then
                PORT_OWNER_RUN_ID=$(cat "$port_file")
                if [ "$PORT_OWNER_RUN_ID" = "$CURRENT_RUN_ID" ]; then
                  # Extract port number from filename
                  PORT=$(basename "$port_file" | sed 's/port-\([0-9]*\)\.run_id/\1/')
                  PID_FILE="$PORT_INFO_DIR/port-${PORT}.pid"
                  
                  # Kill process if PID file exists
                  if [ -f "$PID_FILE" ]; then
                    PID=$(cat "$PID_FILE")
                    if ps -p "$PID" > /dev/null 2>&1; then
                      echo "🛑 Killing process $PID on port $PORT (run_id: $CURRENT_RUN_ID)"
                      pkill -P "$PID" 2>/dev/null || true
                      kill "$PID" 2>/dev/null || true
                      KILLED_COUNT=$((KILLED_COUNT + 1))
                    fi
                  fi
                  
                  # Also kill any process on this port (double-check)
                  PORT_PID=$(lsof -ti:${PORT} 2>/dev/null | head -1)
                  if [ -n "$PORT_PID" ]; then
                    echo "🛑 Killing process $PORT_PID on port $PORT"
                    pkill -P "$PORT_PID" 2>/dev/null || true
                    kill "$PORT_PID" 2>/dev/null || true
                  fi
                  
                  # Clean up orphaned Chrome processes
                  ALL_PORT_PIDS=$(lsof -ti:${PORT} 2>/dev/null || true)
                  if [ -n "$ALL_PORT_PIDS" ]; then
                    for port_pid in $ALL_PORT_PIDS; do
                      if ps -p "$port_pid" -o comm= 2>/dev/null | grep -qE "chrome|chromium"; then
                        echo "🛑 Killing Chrome process $port_pid (using port $PORT)"
                        pkill -P "$port_pid" 2>/dev/null || true
                        kill "$port_pid" 2>/dev/null || true
                      fi
                    done
                  fi
                  
                  # Clean up port info files
                  rm -f "$port_file" "$PID_FILE"
                fi
              fi
            done
          fi

          # Clean up stale port files older than 24 hours
          find "$PORT_INFO_DIR" -name "*.run_id" -mtime +1 -delete 2>/dev/null || true
          find "$PORT_INFO_DIR" -name "*.pid" -mtime +1 -delete 2>/dev/null || true

          if [ $KILLED_COUNT -eq 0 ]; then
            echo "✅ No processes found for run_id: $CURRENT_RUN_ID"
          else
            echo "✅ Cleaned up $KILLED_COUNT process(es) for run_id: $CURRENT_RUN_ID"
          fi

      - name: Clean Work Directories and Build Artifacts
        run: |
          echo "🧹 Starting comprehensive cleanup (SAFE for parallel jobs)..."
          
          RUNNER_USER=$(whoami)
          HOME_DIR=$(eval echo "~$RUNNER_USER")
          CLEANED_SPACE=0
          
          echo "📊 Disk usage before cleanup:"
          df -h / | grep -v Filesystem
          
          # Helper function to check if directory is in use
          is_directory_in_use() {
            local dir="$1"
            # Check if any processes are using this directory
            if lsof +D "$dir" 2>/dev/null | grep -qE "node|npm|webpack|chrome|cypress"; then
              return 0  # In use
            fi
            return 1  # Not in use
          }
          
          # 1. Clean Go upstream build artifacts (only in old work dirs)
          echo ""
          echo "🗑️  Cleaning Go upstream builds (in work dirs >7 days, not in use)..."
          UPSTREAM_COUNT=0
          find "$HOME_DIR"/actions-runner*/_work -maxdepth 1 -name "odh-dashboard" -type d -mtime +7 2>/dev/null | while read work_dir; do
            if [ -d "$work_dir" ] && ! is_directory_in_use "$work_dir"; then
              UPSTREAM_DIRS=$(find "$work_dir" -type d -path "*/packages/*/upstream" 2>/dev/null || true)
              for dir in $UPSTREAM_DIRS; do
                if [ -d "$dir" ]; then
                  SIZE=$(du -sm "$dir" 2>/dev/null | cut -f1)
                  rm -rf "$dir" 2>/dev/null || true
                  CLEANED_SPACE=$((CLEANED_SPACE + SIZE))
                  UPSTREAM_COUNT=$((UPSTREAM_COUNT + 1))
                fi
              done
            fi
          done
          echo "   ✅ Cleaned $UPSTREAM_COUNT upstream build directories"
          
          # 2. Clean old work directories (>7 days, not in use)
          echo ""
          echo "🗑️  Cleaning old work directories (>7 days, not in use)..."
          OLD_WORK_COUNT=0
          SKIPPED_COUNT=0
          find "$HOME_DIR"/actions-runner*/_work -maxdepth 1 -name "odh-dashboard" -type d -mtime +7 2>/dev/null | while read work_dir; do
            if [ -d "$work_dir" ]; then
              if is_directory_in_use "$work_dir"; then
                echo "   ⏭️  Skipped (in use): $(basename $(dirname $work_dir))"
                SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
              else
                SIZE=$(du -sm "$work_dir" 2>/dev/null | cut -f1)
                rm -rf "$work_dir" 2>/dev/null || true
                CLEANED_SPACE=$((CLEANED_SPACE + SIZE))
                OLD_WORK_COUNT=$((OLD_WORK_COUNT + 1))
                echo "   ✅ Cleaned: $(basename $(dirname $work_dir)) (~${SIZE}MB)"
              fi
            fi
          done
          echo "   ✅ Cleaned $OLD_WORK_COUNT old work directories"
          if [ "$SKIPPED_COUNT" -gt 0 ]; then
            echo "   ⏭️  Skipped $SKIPPED_COUNT directories (in use by parallel jobs)"
          fi
          
          # 3. Clean old runner diagnostic logs (>7 days)
          echo ""
          echo "🗑️  Cleaning old runner logs (>7 days)..."
          LOG_COUNT=$(find "$HOME_DIR"/actions-runner*/_diag -name "*.log" -mtime +7 -type f 2>/dev/null | wc -l)
          find "$HOME_DIR"/actions-runner*/_diag -name "*.log" -mtime +7 -delete 2>/dev/null || true
          echo "   ✅ Cleaned $LOG_COUNT old log files"
          
          # 4. Clean old Cypress artifacts (>7 days only - safe even if parallel jobs running)
          echo ""
          echo "🗑️  Cleaning old Cypress artifacts (>7 days)..."
          ARTIFACT_COUNT=0
          for runner_dir in "$HOME_DIR"/actions-runner*; do
            # Screenshots
            if [ -d "$runner_dir/_work/odh-dashboard/odh-dashboard/packages/cypress/results/screenshots" ]; then
              SCREENSHOTS=$(find "$runner_dir/_work/odh-dashboard/odh-dashboard/packages/cypress/results/screenshots" -type f -mtime +7 2>/dev/null | wc -l)
              find "$runner_dir/_work/odh-dashboard/odh-dashboard/packages/cypress/results/screenshots" -type f -mtime +7 -delete 2>/dev/null || true
              ARTIFACT_COUNT=$((ARTIFACT_COUNT + SCREENSHOTS))
            fi
            # Videos
            if [ -d "$runner_dir/_work/odh-dashboard/odh-dashboard/packages/cypress/results/videos" ]; then
              VIDEOS=$(find "$runner_dir/_work/odh-dashboard/odh-dashboard/packages/cypress/results/videos" -type f -mtime +7 2>/dev/null | wc -l)
              find "$runner_dir/_work/odh-dashboard/odh-dashboard/packages/cypress/results/videos" -type f -mtime +7 -delete 2>/dev/null || true
              ARTIFACT_COUNT=$((ARTIFACT_COUNT + VIDEOS))
            fi
          done
          echo "   ✅ Cleaned $ARTIFACT_COUNT old Cypress artifacts"
          
          # 5. Check disk space after cleanup
          echo ""
          echo "📊 Disk usage after cleanup:"
          df -h / | grep -v Filesystem
          echo ""
          echo "💾 Total space cleaned: ~${CLEANED_SPACE}MB"
          echo "✅ Cleanup complete!"