ci-enhanced.yml

name: Enhanced CI Pipeline

on:
  workflow_dispatch: {}
  push:
    branches: [ main, integrate/mvp, "feat/**", "fix/**" ]
  pull_request:
    branches: [ main, integrate/mvp ]

concurrency:
  group: enhanced-ci-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  checks: write
  pull-requests: write

env:
  GO_VERSION: '1.24'
  REGISTRY: ghcr.io
  IMAGE_NAME: nephoran-intent-operator

jobs:
  # =============================================================================
  # Cross-Platform Testing Matrix
  # =============================================================================
  cross-platform-test:
    name: Cross-Platform Tests
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        go-version: ['1.24']
    runs-on: ${{ matrix.os }}
    timeout-minutes: 35
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ matrix.go-version }}
          check-latest: true
          cache: true
          
      - name: Install dependencies
        run: |
          go mod download
          go mod verify
          
      - name: Setup test environment (Unix)
        if: runner.os != 'Windows'
        run: |
          echo "Setting up test environment for Unix..."
          
          # Create test results directory
          mkdir -p test-results
          
          # Install setup-envtest tool
          echo "Installing setup-envtest tool..."
          go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
          
          # Create local bin directory
          mkdir -p ~/.local/bin
          
          # Download and setup envtest binaries
          echo "Downloading Kubernetes 1.29.0 envtest binaries..."
          if setup-envtest use 1.29.0 --bin-dir ~/.local/bin; then
            KUBEBUILDER_ASSETS=$(setup-envtest use 1.29.0 --bin-dir ~/.local/bin -p path 2>/dev/null)
            if [ -n "$KUBEBUILDER_ASSETS" ] && [ -d "$KUBEBUILDER_ASSETS" ]; then
              echo "✅ KUBEBUILDER_ASSETS path: $KUBEBUILDER_ASSETS"
              echo "KUBEBUILDER_ASSETS=$KUBEBUILDER_ASSETS" >> $GITHUB_ENV
            else
              echo "⚠️ Warning: Could not set KUBEBUILDER_ASSETS. Envtest may not work properly."
              echo "KUBEBUILDER_ASSETS=" >> $GITHUB_ENV
            fi
          else
            echo "⚠️ Warning: setup-envtest failed. Tests that don't require envtest will still run."
            echo "KUBEBUILDER_ASSETS=" >> $GITHUB_ENV
          fi
          
      - name: Setup test environment (Windows)
        if: runner.os == 'Windows'
        shell: pwsh
        run: |
          Write-Host "Setting up test environment for Windows..."
          
          # Create test results directory
          New-Item -ItemType Directory -Path test-results -Force | Out-Null
          
          # Install setup-envtest tool
          Write-Host "Installing setup-envtest tool..."
          go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
          
          # Add Go bin to PATH for this session
          $goPath = go env GOPATH
          $goBin = Join-Path $goPath "bin"
          $env:PATH = "$goBin;$env:PATH"
          
          # Create local bin directory
          $localBin = Join-Path $env:USERPROFILE ".local" "bin"
          New-Item -ItemType Directory -Path $localBin -Force | Out-Null
          
          # Download and setup envtest binaries
          Write-Host "Downloading Kubernetes 1.29.0 envtest binaries..."
          try {
            # First attempt: download the binaries
            $output = & setup-envtest.exe use 1.29.0 --bin-dir $localBin 2>&1
            if ($LASTEXITCODE -ne 0) {
              Write-Host "Warning: First setup-envtest attempt failed with exit code $LASTEXITCODE"
              Write-Host "Output: $output"
            }
            
            # Second attempt: get the path to the binaries
            $kubeAssets = & setup-envtest.exe use 1.29.0 --bin-dir $localBin -p path 2>&1
            if ($LASTEXITCODE -eq 0 -and $kubeAssets -and $kubeAssets -notmatch "Error" -and (Test-Path $kubeAssets)) {
              Write-Host "✅ KUBEBUILDER_ASSETS path: $kubeAssets"
              echo "KUBEBUILDER_ASSETS=$kubeAssets" >> $env:GITHUB_ENV
            } else {
              Write-Host "⚠️ Warning: Could not set KUBEBUILDER_ASSETS. Envtest may not work properly."
              Write-Host "setup-envtest output: $kubeAssets"
              # Don't fail the build, as some tests may not require envtest
              echo "KUBEBUILDER_ASSETS=" >> $env:GITHUB_ENV
            }
          } catch {
            Write-Host "⚠️ Warning: Exception during envtest setup: $($_.Exception.Message)"
            Write-Host "Tests that don't require envtest will still run."
            echo "KUBEBUILDER_ASSETS=" >> $env:GITHUB_ENV
          }
          
      - name: Run tests with retry (Unix)
        if: runner.os != 'Windows'
        env:
          USE_EXISTING_CLUSTER: false
          ENVTEST_K8S_VERSION: 1.29.0
          GOMAXPROCS: 2
          GOTRACEBACK: all
          # Note: CGO is required for -race flag, but we disable CGO for static builds
          CGO_ENABLED: 0
        run: |
          # Test with retry for flaky tests
          test_passed=false
          for attempt in 1 2; do
            echo "=== Test attempt $attempt of 2 ==="
            echo "🔧 Running tests without -race flag due to CGO_ENABLED=0"
            echo "📋 Command: go test -v -timeout=30m -count=1 -coverprofile=test-results/coverage-$attempt.out -covermode=atomic ./cmd/conductor-loop ./internal/loop"
            
            # Run tests on all packages with coverage
            # Note: -race flag requires CGO_ENABLED=1, we use CGO_ENABLED=0 for static builds
            if go test -v -timeout=30m -count=1 \
              -coverprofile=test-results/coverage-$attempt.out \
              -covermode=atomic \
              ./cmd/conductor-loop ./internal/loop \
              2>&1 | tee test-results/test-attempt-$attempt.log; then
              echo "✅ Tests passed on attempt $attempt"
              test_passed=true
              
              # Verify coverage file was generated with expected name
              coverage_file="test-results/coverage-$attempt.out"
              if [ -f "$coverage_file" ]; then
                cp "$coverage_file" test-results/coverage.out
                echo "✅ Coverage file generated and copied: $coverage_file -> test-results/coverage.out"
                
                # Show file size for verification
                ls -la "$coverage_file" test-results/coverage.out
              else
                echo "❌ ERROR: Expected coverage file '$coverage_file' was not generated despite tests passing!"
                echo "📁 Files in test-results directory:"
                ls -la test-results/ || echo "test-results directory not found"
                exit 1
              fi
              break
            else
              exit_code=$?
              echo "⚠️ Test attempt $attempt failed with exit code $exit_code"
              
              # Check if partial coverage was generated even though tests failed
              coverage_file="test-results/coverage-$attempt.out"
              if [ -f "$coverage_file" ]; then
                echo "📊 Partial coverage file found despite test failure: $coverage_file"
                cp "$coverage_file" test-results/coverage-partial-$attempt.out
                # Keep the last attempt's coverage as the main one
                cp "$coverage_file" test-results/coverage.out
                echo "💾 Preserved partial coverage for analysis"
              else
                echo "📁 No coverage file generated. Files in test-results:"
                ls -la test-results/ || echo "test-results directory not found"
              fi
              
              if [ $attempt -eq 2 ]; then
                echo "❌ Tests failed after 2 attempts"
                cat test-results/test-attempt-*.log > test-results/combined-test.log
                # Exit with failure after preserving any partial coverage
                exit 1
              else
                echo "⚠️ Retrying in 10s..."
                sleep 10
              fi
            fi
          done
          
          # Verify coverage file exists before processing
          if [ ! -f test-results/coverage.out ]; then
            echo "❌ ERROR: No coverage.out file found after successful tests!"
            exit 1
          fi
          
          # Generate coverage report
          echo "📊 Generating coverage reports..."
          go tool cover -html=test-results/coverage.out -o test-results/coverage.html || echo "Warning: Failed to generate HTML coverage"
          go tool cover -func=test-results/coverage.out > test-results/coverage-summary.txt || echo "Warning: Failed to generate coverage summary"
          
          # Display coverage percentage
          if [ -f test-results/coverage-summary.txt ]; then
            echo "📈 Coverage Summary:"
            tail -5 test-results/coverage-summary.txt
          fi
          
      - name: Run tests with retry (Windows)
        if: runner.os == 'Windows'
        env:
          USE_EXISTING_CLUSTER: false
          ENVTEST_K8S_VERSION: 1.29.0
          GOMAXPROCS: 2
          GOTRACEBACK: all
          # Note: CGO is required for -race flag, but we disable CGO for static builds
          CGO_ENABLED: 0
        run: |
          # Test with retry for flaky tests on Windows
          $success = $false
          for ($attempt = 1; $attempt -le 2; $attempt++) {
            Write-Host "=== Test attempt $attempt of 2 ==="
            Write-Host "🔧 Running tests without -race flag due to CGO_ENABLED=0"
            Write-Host "📋 Command: go test -v -timeout=30m -count=1 -coverprofile=\"test-results/coverage-$attempt.out\" -covermode=atomic ./cmd/conductor-loop ./internal/loop"
            try {
              # Run tests on all packages with coverage
              # Note: -race flag requires CGO_ENABLED=1, we use CGO_ENABLED=0 for static builds
              $testResult = go test -v -timeout=30m -count=1 -coverprofile="test-results/coverage-$attempt.out" -covermode=atomic ./cmd/conductor-loop ./internal/loop 2>&1
              $testResult | Tee-Object test-results/test-attempt-$attempt.log
              if ($LASTEXITCODE -eq 0) {
                Write-Host "✅ Tests passed on attempt $attempt"
                
                # Verify coverage file was generated with expected name
                $coverageFile = "test-results/coverage-$attempt.out"
                if (Test-Path $coverageFile) {
                  Copy-Item $coverageFile test-results/coverage.out -Force
                  Write-Host "✅ Coverage file generated and copied: $coverageFile -> test-results/coverage.out"
                  
                  # Show file info for verification
                  Get-ChildItem $coverageFile, test-results/coverage.out | Format-Table Name, Length, LastWriteTime
                } else {
                  Write-Host "❌ ERROR: Expected coverage file '$coverageFile' was not generated despite tests passing!"
                  Write-Host "📁 Files in test-results directory:"
                  if (Test-Path "test-results") {
                    Get-ChildItem test-results/ | Format-Table Name, Length, LastWriteTime
                  } else {
                    Write-Host "test-results directory not found"
                  }
                  exit 1
                }
                $success = $true
                break
              } else {
                Write-Host "⚠️ Test attempt $attempt failed with exit code $LASTEXITCODE"
                
                # Check if partial coverage was generated even though tests failed
                $coverageFile = "test-results/coverage-$attempt.out"
                if (Test-Path $coverageFile) {
                  Write-Host "📊 Partial coverage file found despite test failure: $coverageFile"
                  Copy-Item $coverageFile test-results/coverage-partial-$attempt.out -Force
                  # Keep the last attempt's coverage as the main one
                  Copy-Item $coverageFile test-results/coverage.out -Force
                  Write-Host "💾 Preserved partial coverage for analysis"
                } else {
                  Write-Host "📁 No coverage file generated. Files in test-results:"
                  if (Test-Path "test-results") {
                    Get-ChildItem test-results/ | Format-Table Name, Length, LastWriteTime
                  } else {
                    Write-Host "test-results directory not found"
                  }
                }
                
                if ($attempt -eq 2) {
                  Write-Host "❌ Tests failed after 2 attempts"
                  Get-Content test-results/test-attempt-*.log | Out-File test-results/combined-test.log
                  # Exit with failure after preserving any partial coverage
                  exit 1
                } else {
                  Write-Host "⚠️ Retrying in 10s..."
                  Start-Sleep 10
                }
              }
            } catch {
              Write-Host "Error in test attempt $attempt: $($_.Exception.Message)"
              
              # Check for partial coverage even on exception
              $coverageFile = "test-results/coverage-$attempt.out"
              if (Test-Path $coverageFile) {
                Write-Host "📊 Partial coverage file found despite error: $coverageFile"
                Copy-Item $coverageFile test-results/coverage-partial-$attempt.out -Force -ErrorAction SilentlyContinue
                Copy-Item $coverageFile test-results/coverage.out -Force -ErrorAction SilentlyContinue
                Write-Host "💾 Preserved partial coverage for analysis"
              }
              
              if ($attempt -eq 2) { exit 1 }
              Start-Sleep 10
            }
          }
          
          # Verify coverage file exists before processing
          if (-not (Test-Path test-results/coverage.out)) {
            Write-Host "❌ ERROR: No coverage.out file found after successful tests!"
            exit 1
          }
          
          # Generate coverage report
          Write-Host "📊 Generating coverage reports..."
          try {
            go tool cover -html=test-results/coverage.out -o test-results/coverage.html
          } catch {
            Write-Host "Warning: Failed to generate HTML coverage"
          }
          try {
            go tool cover -func=test-results/coverage.out | Out-File test-results/coverage-summary.txt
          } catch {
            Write-Host "Warning: Failed to generate coverage summary"
          }
          
          # Display coverage percentage
          if (Test-Path test-results/coverage-summary.txt) {
            Write-Host "📈 Coverage Summary:"
            Get-Content test-results/coverage-summary.txt -Tail 5
          }
          
      - name: Collect CI Diagnostics on Failure (Unix)
        if: failure() && runner.os != 'Windows'
        run: |
          echo "📋 Collecting diagnostics for failed test run..."
          mkdir -p ci-diagnostics
          
          # Copy test results if they exist
          if [ -d "test-results" ]; then
            cp -r test-results ci-diagnostics/ 2>/dev/null || true
          fi
          
          # Collect environment information
          echo "=== Environment Info ===" > ci-diagnostics/environment.txt
          echo "OS: ${{ runner.os }}" >> ci-diagnostics/environment.txt
          echo "Go Version: ${{ matrix.go-version }}" >> ci-diagnostics/environment.txt
          echo "Runner: ${{ matrix.os }}" >> ci-diagnostics/environment.txt
          echo "Timestamp: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> ci-diagnostics/environment.txt
          echo "GitHub Run ID: ${{ github.run_id }}" >> ci-diagnostics/environment.txt
          echo "GitHub Run Number: ${{ github.run_number }}" >> ci-diagnostics/environment.txt
          go version >> ci-diagnostics/environment.txt 2>&1 || true
          go env >> ci-diagnostics/environment.txt 2>&1 || true
          
          # Collect go module information
          echo "=== Go Modules ===" > ci-diagnostics/go-modules.txt
          go list -m all >> ci-diagnostics/go-modules.txt 2>&1 || true
          
          # System diagnostics
          echo "=== System Info ===" > ci-diagnostics/system.txt
          uname -a >> ci-diagnostics/system.txt 2>&1 || true
          df -h >> ci-diagnostics/system.txt 2>&1 || true
          if [ "${{ runner.os }}" = "Linux" ]; then
            free -h >> ci-diagnostics/system.txt 2>&1 || true
          fi
          
          echo "✅ Diagnostics collected in ci-diagnostics/"
          ls -la ci-diagnostics/ || true
        shell: bash
        continue-on-error: true
        
      - name: Collect CI Diagnostics on Failure (Windows)
        if: failure() && runner.os == 'Windows'
        shell: pwsh
        run: |
          Write-Host "📋 Collecting diagnostics for failed test run..."
          New-Item -ItemType Directory -Path ci-diagnostics -Force | Out-Null
          
          # Copy test results if they exist
          if (Test-Path "test-results") {
            Copy-Item -Path test-results -Destination ci-diagnostics/ -Recurse -Force -ErrorAction SilentlyContinue
          }
          
          # Collect environment information
          @"
=== Environment Info ===
OS: ${{ runner.os }}
Go Version: ${{ matrix.go-version }}
Runner: ${{ matrix.os }}
Timestamp: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss UTC')
GitHub Run ID: ${{ github.run_id }}
GitHub Run Number: ${{ github.run_number }}
"@ | Out-File ci-diagnostics/environment.txt
          
          go version 2>&1 | Out-File ci-diagnostics/environment.txt -Append
          go env 2>&1 | Out-File ci-diagnostics/environment.txt -Append
          
          # Collect go module information
          "=== Go Modules ===" | Out-File ci-diagnostics/go-modules.txt
          go list -m all 2>&1 | Out-File ci-diagnostics/go-modules.txt -Append
          
          # System diagnostics
          @"
=== System Info ===
Computer Name: $env:COMPUTERNAME
OS Version: $([System.Environment]::OSVersion.VersionString)
Processor Count: $([System.Environment]::ProcessorCount)
"@ | Out-File ci-diagnostics/system.txt
          
          Get-PSDrive -PSProvider FileSystem | Out-File ci-diagnostics/system.txt -Append
          
          Write-Host "✅ Diagnostics collected in ci-diagnostics/"
          Get-ChildItem ci-diagnostics/ -ErrorAction SilentlyContinue
        continue-on-error: true
          
      - name: Upload test artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-results-${{ matrix.os }}-${{ github.run_id }}
          path: |
            test-results/
            ci-diagnostics/
          retention-days: 30
          if-no-files-found: warn
          
  # =============================================================================
  # Build Verification
  # =============================================================================
  build-verification:
    name: Build Verification
    runs-on: ubuntu-latest
    needs: cross-platform-test
    timeout-minutes: 20
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          check-latest: true
          cache: true
          
      - name: Download dependencies
        run: |
          go mod download
          go mod verify
          
      - name: Build all binaries
        run: |
          mkdir -p bin
          
          # Build all cmd binaries
          for cmd_dir in cmd/*/; do
            if [ -f "$cmd_dir/main.go" ]; then
              cmd_name=$(basename "$cmd_dir")
              echo "Building $cmd_name..."
              go build -v -o "bin/$cmd_name" "./$cmd_dir"
            fi
          done
          
          ls -la bin/
          
      - name: Upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: build-artifacts-${{ github.run_id }}
          path: bin/
          retention-days: 7

  # =============================================================================
  # Security and Quality Gates
  # =============================================================================
  security-scan:
    name: Security Scan
    runs-on: ubuntu-latest
    needs: cross-platform-test
    timeout-minutes: 15
    continue-on-error: true
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: true
          
      - name: Run security scans
        run: |
          mkdir -p security-reports
          
          # Install security tools
          go install golang.org/x/vuln/cmd/govulncheck@latest
          go install github.com/securecode/gosec/v2/cmd/gosec@latest || echo "gosec install failed, continuing..."
          
          # Run vulnerability check
          echo "Running govulncheck..."
          govulncheck -json ./... > security-reports/vulns.json 2>&1 || echo "Vulnerabilities found"
          
          # Run gosec if available
          if command -v gosec &> /dev/null; then
            echo "Running gosec..."
            gosec -fmt sarif -out security-reports/gosec.sarif ./... || echo "Security issues found"
          fi
          
      - name: Upload security reports
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: security-reports-${{ github.run_id }}
          path: security-reports/
          retention-days: 30

  # =============================================================================
  # Coverage Aggregation
  # =============================================================================
  coverage-report:
    name: Coverage Report
    runs-on: ubuntu-latest
    needs: cross-platform-test
    if: always()
    timeout-minutes: 10
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          
      - name: Download test artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: test-results-*-${{ github.run_id }}
          path: all-test-results/
          merge-multiple: true
          
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: true
          
      - name: Download Go dependencies
        run: |
          go mod download
          go mod verify
          
      - name: Aggregate coverage
        run: |
          mkdir -p aggregated-coverage
          
          # Find and combine coverage files
          find all-test-results -name "coverage.out" -type f | head -1 | while read coverage_file; do
            if [ -f "$coverage_file" ]; then
              cp "$coverage_file" aggregated-coverage/coverage.out
              
              # Generate coverage reports
              go tool cover -html=aggregated-coverage/coverage.out -o aggregated-coverage/coverage.html
              go tool cover -func=aggregated-coverage/coverage.out > aggregated-coverage/coverage-summary.txt
              
              # Extract coverage percentage
              coverage_pct=$(tail -1 aggregated-coverage/coverage-summary.txt | awk '{print $NF}' || echo "0%")
              echo "Coverage: $coverage_pct"
              echo "COVERAGE_PERCENTAGE=$coverage_pct" >> $GITHUB_ENV
            fi
          done
          
      - name: Upload coverage report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report-${{ github.run_id }}
          path: aggregated-coverage/
          retention-days: 30

  # =============================================================================
  # Final Status Check
  # =============================================================================
  ci-status:
    name: CI Status Check
    runs-on: ubuntu-latest
    needs: [cross-platform-test, build-verification, security-scan, coverage-report]
    if: always()
    timeout-minutes: 5
    
    steps:
      - name: Check results
        run: |
          echo "=== Enhanced CI Pipeline Results ==="
          echo "Cross-Platform Test: ${{ needs.cross-platform-test.result }}"
          echo "Build Verification: ${{ needs.build-verification.result }}"
          echo "Security Scan: ${{ needs.security-scan.result }}"
          echo "Coverage Report: ${{ needs.coverage-report.result }}"
          
          # Fail if critical jobs failed
          if [[ "${{ needs.cross-platform-test.result }}" == "failure" ]]; then
            echo "❌ Cross-platform tests failed"
            exit 1
          fi
          
          if [[ "${{ needs.build-verification.result }}" == "failure" ]]; then
            echo "❌ Build verification failed"
            exit 1
          fi
          
          echo "✅ Enhanced CI Pipeline Completed"
          
      - name: Generate summary
        if: always()
        run: |
          echo "## 🚀 Enhanced CI Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Component | Status | Description |" >> $GITHUB_STEP_SUMMARY
          echo "|-----------|--------|-------------|" >> $GITHUB_STEP_SUMMARY
          echo "| 🧪 Cross-Platform Tests | ${{ needs.cross-platform-test.result }} | Tests across Ubuntu, Windows, macOS |" >> $GITHUB_STEP_SUMMARY
          echo "| 🔨 Build Verification | ${{ needs.build-verification.result }} | Binary compilation verification |" >> $GITHUB_STEP_SUMMARY
          echo "| 🔒 Security Scan | ${{ needs.security-scan.result }} | Vulnerability and security checks |" >> $GITHUB_STEP_SUMMARY
          echo "| 📊 Coverage Report | ${{ needs.coverage-report.result }} | Test coverage aggregation |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### ⏱️ Performance Metrics" >> $GITHUB_STEP_SUMMARY
          echo "- **Workflow**: ${{ github.workflow }}" >> $GITHUB_STEP_SUMMARY
          echo "- **Run ID**: ${{ github.run_id }}" >> $GITHUB_STEP_SUMMARY
          echo "- **Commit**: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY