openvx-conformance.yml

name: OpenVX Vision Conformance

on:
  push:
    branches: [ main, master, develop ]
  pull_request:
    branches: [ main, master ]
  workflow_dispatch:
    inputs:
      test_filter:
        description: 'Test filter pattern (optional)'
        required: false
        default: ''
      verbose:
        description: 'Verbose output'
        required: false
        default: 'false'

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true

jobs:
  build-and-test:
    runs-on: ubuntu-22.04
    timeout-minutes: 360
    
    steps:
    - name: Checkout rustVX
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        submodules: recursive
    
    - name: Debug directory structure
      run: |
        echo "=== Directory Structure ==="
        ls -la
        echo "=== OpenVX-cts structure ==="
        ls -la OpenVX-cts/ || echo "No OpenVX-cts directory!"
        echo "=== Current working directory ==="
        pwd
    
    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y \
          build-essential \
          cmake \
          ninja-build \
          libgtest-dev \
          libopencv-dev \
          python3 \
          python3-pip
    
    - name: Install Rust
      run: |
        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
        source $HOME/.cargo/env
        rustc --version
        cargo --version
    
    - name: Build rustVX
      run: |
        source $HOME/.cargo/env
        echo "Building rustVX..."
        cargo build --release
        echo "Build complete. Library files:"
        ls -la target/release/*.so target/release/*.rlib 2>/dev/null || true
    
    - name: Build OpenVX CTS
      run: |
        echo "=== Building OpenVX CTS ==="
        
        # Check OpenVX-cts directory
        if [ ! -d "OpenVX-cts" ]; then
          echo "ERROR: OpenVX-cts directory not found!"
          ls -la
          exit 1
        fi
        
        cd OpenVX-cts
        
        # Check for CMakeLists.txt
        if [ ! -f "CMakeLists.txt" ]; then
          echo "ERROR: CMakeLists.txt not found!"
          ls -la
          exit 1
        fi
        
        # Prepare include directory
        mkdir -p include
        if [ -d "../include" ]; then
          echo "Copying OpenVX headers..."
          cp -r ../include/* include/ 2>/dev/null || true
        fi
        
        mkdir -p build
        cd build
        
        # Configure with verbose output and proper include paths
        echo "Running cmake..."
        cmake .. \
          -DCMAKE_BUILD_TYPE=Release \
          -DCMAKE_C_STANDARD_LIBRARIES="-lm" \
          -DCMAKE_CXX_STANDARD_LIBRARIES="-lm" \
          -DOPENVX_INCLUDES="${{ github.workspace }}/include;${{ github.workspace }}/OpenVX-cts/include" \
          -DOPENVX_LIBRARIES="${{ github.workspace }}/target/release/libopenvx_ffi.so;m" \
          -DOPENVX_CONFORMANCE_VISION=ON \
          2>&1 || {
          echo "CMAKE FAILED!"
          echo "CMake output:"
          cat CMakeFiles/CMakeOutput.log 2>/dev/null | tail -100 || echo "No CMakeOutput.log"
          cat CMakeFiles/CMakeError.log 2>/dev/null | tail -100 || echo "No CMakeError.log"
          exit 1
        }
        
        # Build
        echo "Running make..."
        make VERBOSE=1 -j$(nproc) 2>&1 || {
          echo "MAKE FAILED!"
          exit 1
        }
        
        echo "Build complete. Listing binaries:"
        ls -la bin/ 2>/dev/null || echo "No bin directory"
        find . -name "vx_test_conformance" -type f 2>/dev/null
    
    - name: Run Vision Conformance Tests
      id: run_tests
      continue-on-error: true
      run: |
        source $HOME/.cargo/env
        echo "========================================"
        echo "Running OpenVX Vision Conformance Tests"
        echo "========================================"
        
        # Find CTS build directory
        CTS_DIR="OpenVX-cts/build"
        if [ ! -d "$CTS_DIR" ]; then
          echo "ERROR: CTS build directory not found!"
          exit 1
        fi
        
        cd "$CTS_DIR"
        
        # Check for test binary
        if [ ! -f "bin/vx_test_conformance" ]; then
          echo "Test binary not found, searching..."
          find . -name "vx_test_conformance" -type f 2>/dev/null
          ls -la bin/ 2>/dev/null || true
        fi
        
        # Set library path for the conformance tests
        export LD_LIBRARY_PATH=${{ github.workspace }}/target/release:$LD_LIBRARY_PATH
        export VX_TEST_DATA_PATH="${{ github.workspace }}/OpenVX-cts/test_data/"
        
        echo "Library path: $LD_LIBRARY_PATH"
        echo "VX_TEST_DATA_PATH: $VX_TEST_DATA_PATH"
        
        # Create output directory for results
        mkdir -p ${{ github.workspace }}/test-results
        
        # Run tests - extended timeout for full vision conformance (~15k tests)
        timeout 21000 ./bin/vx_test_conformance 2>&1 | tee ${{ github.workspace }}/test-results/vision_test_output.txt || true
        
        echo "Test execution completed."
    
    - name: Analyze test results
      id: analyze_results
      run: |
        source $HOME/.cargo/env
        echo "========================================"
        echo "Analyzing Test Results"
        echo "========================================"
        
        OUTPUT_FILE="${{ github.workspace }}/test-results/vision_test_output.txt"
        
        # Create output file if it doesn't exist
        mkdir -p ${{ github.workspace }}/test-results
        touch "$OUTPUT_FILE"
        
        # Count passing tests
        PASS_COUNT=$(grep -c "\[     DONE \]" "$OUTPUT_FILE" 2>/dev/null || echo "0")
        echo "Tests passed: $PASS_COUNT"
        
        # Count failing tests
        FAIL_COUNT=$(grep -c "\[ !FAILED! \]" "$OUTPUT_FILE" 2>/dev/null || echo "0")
        echo "Tests failed: $FAIL_COUNT"
        
        # Extract test summary
        echo "========================================" | tee ${{ github.workspace }}/test-results/summary.txt
        echo "Vision Conformance Test Summary" | tee -a ${{ github.workspace }}/test-results/summary.txt
        echo "========================================" | tee -a ${{ github.workspace }}/test-results/summary.txt
        echo "Passed: $PASS_COUNT" | tee -a ${{ github.workspace }}/test-results/summary.txt
        echo "Failed: $FAIL_COUNT" | tee -a ${{ github.workspace }}/test-results/summary.txt
        echo "Total: $((PASS_COUNT + FAIL_COUNT))" | tee -a ${{ github.workspace }}/test-results/summary.txt
        
        if [ "$FAIL_COUNT" -eq 0 ] && [ "$PASS_COUNT" -gt 0 ]; then
          echo "Status: ✅ ALL TESTS PASSED" | tee -a ${{ github.workspace }}/test-results/summary.txt
        elif [ "$PASS_COUNT" -gt 0 ]; then
          echo "Status: ⚠️ PARTIAL SUCCESS ($PASS_COUNT/$((PASS_COUNT + FAIL_COUNT)))" | tee -a ${{ github.workspace }}/test-results/summary.txt
        else
          echo "Status: ❌ NO TESTS PASSED" | tee -a ${{ github.workspace }}/test-results/summary.txt
        fi
        
        # Set outputs for GitHub Actions summary
        echo "pass_count=$PASS_COUNT" >> $GITHUB_OUTPUT
        echo "fail_count=$FAIL_COUNT" >> $GITHUB_OUTPUT
    
    - name: Check baseline test regressions
      run: |
        echo "========================================"
        echo "Checking Baseline Test Regressions"
        echo "========================================"
        
        OUTPUT_FILE="${{ github.workspace }}/test-results/vision_test_output.txt"
        
        # Baseline test categories that must all pass
        BASELINE_CATEGORIES=("GraphBase" "SmokeTestBase" "TargetBase" "Logging")
        REGRESSION_FOUND=0
        
        for category in "${BASELINE_CATEGORIES[@]}"; do
          # Count passed and failed tests in this category
          PASSED=$(grep -c "\[     DONE \] ${category}" "$OUTPUT_FILE" 2>/dev/null || echo "0")
          FAILED=$(grep -c "\[ !FAILED! \] ${category}" "$OUTPUT_FILE" 2>/dev/null || echo "0")
          
          echo "${category}: ${PASSED} passed, ${FAILED} failed"
          
          if [ "$FAILED" -gt 0 ]; then
            echo "❌ REGRESSION: ${category} has ${FAILED} failing tests"
            REGRESSION_FOUND=1
            # Show which tests failed
            grep "\[ !FAILED! \] ${category}" "$OUTPUT_FILE" 2>/dev/null | head -10
          fi
        done
        
        echo "========================================"
        if [ "$REGRESSION_FOUND" -eq 1 ]; then
          echo "❌ FAILURE: Baseline test regressions detected!"
          echo "The following categories must have 0 failures:"
          echo "  - GraphBase (14 tests)"
          echo "  - SmokeTestBase (7 tests)"
          echo "  - TargetBase (3 tests)"
          echo "  - Logging (1 test)"
          exit 1
        else
          echo "✅ All baseline tests passing (GraphBase, SmokeTestBase, TargetBase, Logging)"
        fi
        echo "========================================"

    - name: Upload test results
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: conformance-test-results
        path: |
          test-results/
        retention-days: 30
    
    - name: Create GitHub Actions Summary
      if: always()
      run: |
        echo "## 🧪 OpenVX Vision Conformance Test Results" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        
        if [ -f "test-results/summary.txt" ]; then
          cat test-results/summary.txt >> $GITHUB_STEP_SUMMARY
        fi
        
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "### Build Information" >> $GITHUB_STEP_SUMMARY
        source $HOME/.cargo/env
        echo "- **Rust Version:** $(rustc --version)" >> $GITHUB_STEP_SUMMARY
        echo "- **Build Type:** Release" >> $GITHUB_STEP_SUMMARY
        echo "- **CTS Build:** OpenVX Conformance Test Suite" >> $GITHUB_STEP_SUMMARY
        
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "📥 **Download full logs:** See Artifacts section above" >> $GITHUB_STEP_SUMMARY