infra-verify.yml

name: Infrastructure Verification

on:
  workflow_dispatch:
  pull_request:
    types: [opened, synchronize, reopened]
  merge_group:
    types: [checks_requested]

# Cancel previous runs when a PR is updated
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

jobs:
  infra-verify:
    name: Infrastructure Verification
    runs-on: [self-hosted, enclave-small]
    timeout-minutes: 120

    env:
      DEV_SCRIPTS_PATH: ${{ vars.DEV_SCRIPTS_PATH }}
      BASE_WORKING_DIR: ${{ vars.BASE_WORKING_DIR }}
      PULL_SECRET: ${{ secrets.PULL_SECRET }}
      # Bypass CI_TOKEN requirement (we only use dev-scripts for infra, not cluster install)
      OPENSHIFT_CI: "true"
      ENCLAVE_ENABLE_GPU_PASSTHROUGH: "false"

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Generate unique cluster name
        id: cluster_name
        uses: ./.github/actions/setup-cluster-name
        with:
          naming-strategy: hash
          prefix: eci
          run-id: ${{ github.run_id }}

      - name: Export cluster name to environment
        run: |
          echo "ENCLAVE_CLUSTER_NAME=${{ steps.cluster_name.outputs.cluster-name }}" >> $GITHUB_ENV
          echo "Cluster name for this workflow: ${{ steps.cluster_name.outputs.cluster-name }}"

      - name: Setup cluster-specific working directory
        env:
          BASE_WORKING_DIR: ${{ env.BASE_WORKING_DIR }}
        run: |
          echo "Using cluster name: ${ENCLAVE_CLUSTER_NAME}"
          ./scripts/setup/setup_working_dir.sh
          echo "WORKING_DIR=$(cat /tmp/working_dir)" >> $GITHUB_ENV

          # Create directory for step logs
          mkdir -p step-logs
          echo "STEP_LOGS_DIR=$PWD/step-logs" >> $GITHUB_ENV

      - name: Workflow information
        env:
          PR_TITLE: ${{ github.event.pull_request.title }}
        run: |
          echo "## Workflow Information" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- **Trigger**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
          echo "- **Cleanup Strategy**: $CLEANUP_STRATEGY" >> $GITHUB_STEP_SUMMARY
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            echo "- **PR Number**: #${{ github.event.pull_request.number }}" >> $GITHUB_STEP_SUMMARY
            echo "- **PR Title**: $PR_TITLE" >> $GITHUB_STEP_SUMMARY
          fi
          echo "" >> $GITHUB_STEP_SUMMARY

      - name: Pre-flight checks
        uses: ./.github/actions/preflight-checks
        with:
          title: Infrastructure Verification Pre-flight Checks
          check-system-resources: 'true'
          check-libvirt: 'true'

      - name: Allocate unique subnet for cluster
        id: allocate_subnet
        uses: ./.github/actions/allocate-subnet

      - name: Save allocation log
        if: always() && steps.allocate_subnet.outcome != 'skipped'
        run: |
          echo "Subnet allocation completed with status: ${{ steps.allocate_subnet.outcome }}" > step-logs/01-allocate-subnet.log

      - name: Create infrastructure
        id: create_infra
        env:
          WORKING_DIR: ${{ env.WORKING_DIR }}
        run: |
          echo "## Creating Infrastructure" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Creating VMs, networks, and BMC emulation..." >> $GITHUB_STEP_SUMMARY
          echo "Working Directory: ${WORKING_DIR:-not-set}" >> $GITHUB_STEP_SUMMARY

          # Initialize log file with header
          echo "=== Infrastructure Setup Log ===" > step-logs/02-setup-infrastructure.log
          echo "Started: $(date)" >> step-logs/02-setup-infrastructure.log
          echo "" >> step-logs/02-setup-infrastructure.log

          # Run command and capture all output
          set +e
          make -f Makefile.ci environment 2>&1 | tee -a step-logs/02-setup-infrastructure.log
          EXIT_CODE=${PIPESTATUS[0]}
          set -e

          # Log completion
          echo "" >> step-logs/02-setup-infrastructure.log
          echo "Completed: $(date)" >> step-logs/02-setup-infrastructure.log
          echo "Exit code: $EXIT_CODE" >> step-logs/02-setup-infrastructure.log

          # Exit with the original code
          if [ $EXIT_CODE -ne 0 ]; then
            echo "❌ Infrastructure creation failed (exit code: $EXIT_CODE)" >> $GITHUB_STEP_SUMMARY
            exit $EXIT_CODE
          fi

          echo "✅ Infrastructure created" >> $GITHUB_STEP_SUMMARY

      - name: Provision Landing Zone
        id: provision_lz
        env:
          WORKING_DIR: ${{ env.WORKING_DIR }}
        run: |
          echo "## Provisioning Landing Zone" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Installing CentOS Stream and configuring Landing Zone VM..." >> $GITHUB_STEP_SUMMARY
          make -f Makefile.ci provision-landing-zone
          echo "✅ Landing Zone provisioned" >> $GITHUB_STEP_SUMMARY

      - name: Install Enclave Lab (Connected Mode)
        id: install_enclave
        run: |
          echo "## Installing Enclave Lab" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Running in connected mode for faster testing..." >> $GITHUB_STEP_SUMMARY

          ENCLAVE_DEPLOYMENT_MODE=connected make -f Makefile.ci install-enclave 2>&1 | tee step-logs/03-install-enclave.log

      - name: Collect step logs
        if: always()
        env:
          WORKING_DIR: ${{ env.WORKING_DIR }}
        run: make -f Makefile.ci collect-step-logs

      - name: Collect artifacts
        if: always()
        uses: ./.github/actions/collect-artifacts
        with:
          artifact-type: infra
          output-directory: ci-artifacts

      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: infra-verify-artifacts
          path: |
            ci-artifacts/
            step-logs/
          retention-days: 7

      - name: Cleanup infrastructure
        if: always()
        run: |
          echo "## Cleanup Infrastructure" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Run cleanup and capture exit code
          set +e
          make -f Makefile.ci clean 2>&1 | tee step-logs/03-cleanup.log
          CLEANUP_EXIT_CODE=$?
          set -e

          # Report results
          if [ $CLEANUP_EXIT_CODE -eq 0 ]; then
            echo "✅ Cleanup completed successfully" >> $GITHUB_STEP_SUMMARY
          else
            echo "⚠️ Cleanup completed with errors (exit code: $CLEANUP_EXIT_CODE)" >> $GITHUB_STEP_SUMMARY
            echo "Check step-logs/03-cleanup.log for details" >> $GITHUB_STEP_SUMMARY
          fi

      - name: Collect post-cleanup state
        if: always()
        run: |
          mkdir -p cleanup-state/libvirt cleanup-state/system cleanup-logs

          # Copy cleanup log if it exists
          [ -f step-logs/03-cleanup.log ] && cp step-logs/03-cleanup.log cleanup-logs/ || true

          # Capture post-cleanup system state
          sudo virsh pool-list --all --details > cleanup-state/libvirt/storage-pools.txt 2>&1 || true
          sudo virsh net-list --all > cleanup-state/libvirt/networks.txt 2>&1 || true
          sudo virsh list --all > cleanup-state/libvirt/vms.txt 2>&1 || true
          df -h > cleanup-state/system/disk-usage.txt 2>&1 || true

      - name: Verify cleanup completion
        if: always()
        run: |
          echo "## Cleanup Verification" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          # Check for leftover resources
          LEFTOVER_POOLS=$(sudo virsh pool-list --all --name | grep "^eci-" || true)
          LEFTOVER_NETWORKS=$(sudo virsh net-list --all --name | grep "^eci-.*-[pe]$" || true)
          LEFTOVER_VMS=$(sudo virsh list --all --name | grep "^eci-" || true)

          # Save verification results to file (for artifacts)
          {
            echo "=== Cleanup Verification Results ==="
            echo "Timestamp: $(date -u +"%Y%m%d-%H%M%S")"
            echo ""
            if [ -z "$LEFTOVER_POOLS" ] && [ -z "$LEFTOVER_NETWORKS" ] && [ -z "$LEFTOVER_VMS" ]; then
              echo "✅ No leftover resources detected"
            else
              echo "⚠️ Leftover resources detected:"
              [ -n "$LEFTOVER_POOLS" ] && echo "- Pools: $LEFTOVER_POOLS"
              [ -n "$LEFTOVER_NETWORKS" ] && echo "- Networks: $LEFTOVER_NETWORKS"
              [ -n "$LEFTOVER_VMS" ] && echo "- VMs: $LEFTOVER_VMS"
            fi
          } | tee cleanup-logs/verification.txt

          # Also report to step summary
          if [ -z "$LEFTOVER_POOLS" ] && [ -z "$LEFTOVER_NETWORKS" ] && [ -z "$LEFTOVER_VMS" ]; then
            echo "✅ No leftover resources detected" >> $GITHUB_STEP_SUMMARY
          else
            echo "⚠️ **Leftover resources detected:**" >> $GITHUB_STEP_SUMMARY
            [ -n "$LEFTOVER_POOLS" ] && echo "- **Pools**: $(echo $LEFTOVER_POOLS | tr '\n' ' ')" >> $GITHUB_STEP_SUMMARY
            [ -n "$LEFTOVER_NETWORKS" ] && echo "- **Networks**: $(echo $LEFTOVER_NETWORKS | tr '\n' ' ')" >> $GITHUB_STEP_SUMMARY
            [ -n "$LEFTOVER_VMS" ] && echo "- **VMs**: $(echo $LEFTOVER_VMS | tr '\n' ' ')" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "These resources should be cleaned up manually or via the cleanup workflow." >> $GITHUB_STEP_SUMMARY
          fi

      - name: Upload post-cleanup artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: post-cleanup-state
          path: |
            cleanup-logs/
            cleanup-state/
          retention-days: 7

      - name: Workflow summary
        if: always()
        run: |
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "---" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "## Workflow Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- **Trigger**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY
          echo "- **Result**: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY