conformance.yml

name: OpenVX Conformance Tests

on:
  push:
    branches: [master, main, develop]
    paths-ignore:
      - '**/*.md'
      - 'docs/**'
      - 'LICENSE'
  pull_request:
    branches: [master, main, develop]
    paths-ignore:
      - '**/*.md'
      - 'docs/**'
      - 'LICENSE'

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:
  build:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential cmake
      - name: Install Rust
        run: |
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
          source $HOME/.cargo/env
          rustc --version
          cargo --version
      - name: Build rustVX
        run: |
          source $HOME/.cargo/env
          cargo build --release
      - name: Build OpenVX CTS
        run: |
          cd OpenVX-cts
          mkdir -p include
          if [ -d "../include" ]; then
            cp -r ../include/* include/ 2>/dev/null || true
          fi
          mkdir -p build
          cd build
          cmake .. \
            -DCMAKE_BUILD_TYPE=Release \
            -DCMAKE_C_STANDARD_LIBRARIES="-lm" \
            -DCMAKE_CXX_STANDARD_LIBRARIES="-lm" \
            -DOPENVX_INCLUDES="${{ github.workspace }}/include;${{ github.workspace }}/OpenVX-cts/include" \
            -DOPENVX_LIBRARIES="${{ github.workspace }}/target/release/libopenvx_ffi.so;m" \
            -DOPENVX_CONFORMANCE_VISION=ON \
            -DOPENVX_USE_ENHANCED_VISION=ON
          make -j$(nproc)
      - name: Upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: build-artifacts
          # `include/` is bundled so the downstream benchmark job can build
          # openvx-mark against rustVX without needing to check out the
          # rustVX source tree.
          path: |
            target/release/libopenvx_ffi.so
            OpenVX-cts/build/bin/vx_test_conformance
            OpenVX-cts/test_data/
            include/
          retention-days: 1

  # Build the Khronos OpenVX sample implementation in its own phase, in
  # parallel with the rustVX `build` job, and upload the resulting library
  # + headers as a self-contained archive. The benchmark job below pulls
  # both archives down onto a single runner so rustVX and the Khronos
  # sample are exercised on identical hardware.
  build-khronos-sample:
    name: Build Khronos OpenVX sample
    runs-on: ubuntu-22.04
    steps:
      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential cmake git python3
      - name: Build Khronos OpenVX sample
        run: |
          git clone --recursive --depth 1 \
            https://github.com/KhronosGroup/OpenVX-sample-impl.git khronos-sample
          cd khronos-sample
          python3 Build.py --os=Linux --arch=64 --conf=Release
      - name: Stage Khronos sample archive
        run: |
          set -euo pipefail
          LIB_SRC=$(dirname $(find khronos-sample -name "libopenvx.so" -not -path "*/build/*" | head -1))
          echo "Khronos libraries discovered in: $LIB_SRC"
          mkdir -p khronos-stage/lib
          cp "$LIB_SRC"/libopenvx*.so "$LIB_SRC"/libvxu*.so khronos-stage/lib/
          cp -r khronos-sample/api-docs/include khronos-stage/include
          ls -R khronos-stage
      - name: Upload Khronos sample artifacts
        uses: actions/upload-artifact@v4
        with:
          name: khronos-sample-artifacts
          path: khronos-stage/
          retention-days: 1

  baseline:
    runs-on: ubuntu-22.04
    needs: build
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run baseline tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 300 ./bin/vx_test_conformance --filter="GraphBase.*:Logging.*:SmokeTestBase.*:SmokeTest.*:TargetBase.*:Target.*"

  graph:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run graph tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 600 ./bin/vx_test_conformance --filter="Graph.*:GraphCallback.*:GraphDelay.*:GraphROI.*:UserNode.*"

  data-objects:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run data object tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 300 ./bin/vx_test_conformance --filter="Scalar.*:Array.*:ObjectArray.*:Matrix.*:Convolution.*:Distribution.*:LUT.*:Histogram.*"

  image-ops:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run image operation tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 600 ./bin/vx_test_conformance --filter="Image.*:vxCopyImagePatch.*:vxMapImagePatch.*:vxCreateImageFromChannel.*:vxCopyRemapPatch.*:vxMapRemapPatch.*"

  vision-color:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run color and channel tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 300 ./bin/vx_test_conformance --filter="ColorConvert.*:ChannelExtract.*:ChannelCombine.*:vxConvertDepth.*:vxuConvertDepth.*"

  vision-filters:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run filter and morphology tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 600 ./bin/vx_test_conformance --filter="Box3x3.*:Gaussian3x3.*:Median3x3.*:Dilate3x3.*:Erode3x3.*:Sobel3x3.*:Magnitude.*:Phase.*:NonLinearFilter.*:Convolve.*:EqualizeHistogram.*"

  vision-arithmetic:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run arithmetic and bitwise tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 600 ./bin/vx_test_conformance --filter="vxAddSub.*:vxuAddSub.*:vxMultiply.*:vxuMultiply.*:vxBinOp8u.*:vxuBinOp8u.*:vxBinOp16s.*:vxuBinOp16s.*:vxNot.*:vxuNot.*:WeightedAverage.*:Threshold.*"

  vision-geometric:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run geometric transform tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 600 ./bin/vx_test_conformance --filter="Scale.*:WarpAffine.*:WarpPerspective.*:Remap.*:HalfScaleGaussian.*"

  vision-features:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run feature and edge detection tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 600 ./bin/vx_test_conformance --filter="HarrisCorners.*:FastCorners.*:vxCanny.*:vxuCanny.*"

  vision-statistics:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run statistics and analysis tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 300 ./bin/vx_test_conformance --filter="MeanStdDev.*:MinMaxLoc.*:Integral.*"

  vision-pyramid:
    runs-on: ubuntu-22.04
    needs: build
    continue-on-error: true
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run pyramid and optical flow tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 300 ./bin/vx_test_conformance --filter="GaussianPyramid.*:LaplacianPyramid.*:LaplacianReconstruct.*:OptFlowPyrLK.*"

  # Enhanced Vision Phase 1 — only the kernels rustVX has actually
  # implemented from the OpenVX 1.3 Enhanced Vision feature set. The CTS
  # binary is built with `OPENVX_USE_ENHANCED_VISION=ON`, but this job
  # filters strictly to the kernels Phase 1 ships (vxMin / vxMax). The
  # remaining Enhanced Vision symbols are exposed as link stubs in
  # rustVX so the binary can build; they are not exercised here and will
  # be replaced by real kernels in subsequent phases.
  enhanced-vision:
    name: "enhanced-vision (Phase 1 — Min/Max)"
    runs-on: ubuntu-22.04
    needs: build
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
      - name: Run Enhanced Vision Phase 1 tests
        run: |
          chmod +x OpenVX-cts/build/bin/vx_test_conformance
          cd OpenVX-cts/build
          export LD_LIBRARY_PATH=${{ github.workspace }}/target/release
          export VX_TEST_DATA_PATH=${{ github.workspace }}/OpenVX-cts/test_data/
          timeout 120 ./bin/vx_test_conformance --filter="Min.*:Max.*"

  # Performance benchmark using openvx-mark, comparing rustVX against the
  # Khronos OpenVX sample implementation on the SAME runner so the two
  # numbers come from identical hardware. This job does NOT rebuild either
  # implementation — it just downloads the archives produced by the
  # `build` and `build-khronos-sample` phases above, builds the openvx-mark
  # tool against each, runs the same workload, and compares the JSON
  # reports. The CTS jobs above use `continue-on-error: true`, so this
  # job effectively gates on `build`, `build-khronos-sample`, and
  # `baseline` succeeding (matching the existing CTS gate).
  benchmark:
    name: Benchmark & compare (rustVX vs Khronos sample)
    runs-on: ubuntu-22.04
    needs:
      - build
      - build-khronos-sample
      - baseline
      - graph
      - data-objects
      - image-ops
      - vision-color
      - vision-filters
      - vision-arithmetic
      - vision-geometric
      - vision-features
      - vision-statistics
      - vision-pyramid
    continue-on-error: true
    steps:
      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential cmake git python3

      - name: Download rustVX archive
        uses: actions/download-artifact@v4
        with:
          name: build-artifacts
          path: ${{ github.workspace }}/rustvx-pkg

      - name: Download Khronos sample archive
        uses: actions/download-artifact@v4
        with:
          name: khronos-sample-artifacts
          path: ${{ github.workspace }}/khronos-pkg

      - name: Expose rustVX as libopenvx / libvxu
        id: rustvx
        # openvx-mark uses `find_library(NAMES openvx)` and
        # `find_library(NAMES vxu)`. rustVX ships a single
        # `libopenvx_ffi.so` that exports the full set of `vx*`/`vxu*`
        # symbols, so symlink the two classic Khronos library names to
        # it without changing rustVX's own build output.
        run: |
          set -euo pipefail
          LIB_DIR=${{ github.workspace }}/rustvx-pkg/target/release
          chmod -R u+rwX "$LIB_DIR"
          cd "$LIB_DIR"
          ln -sf libopenvx_ffi.so libopenvx.so
          ln -sf libopenvx_ffi.so libvxu.so
          ls -la libopenvx*.so libvxu*.so
          echo "lib_dir=$LIB_DIR" >> "$GITHUB_OUTPUT"
          echo "include_dir=${{ github.workspace }}/rustvx-pkg/include" >> "$GITHUB_OUTPUT"

      - name: Inspect Khronos sample archive
        id: khronos
        run: |
          set -euo pipefail
          LIB_DIR=${{ github.workspace }}/khronos-pkg/lib
          INCLUDE_DIR=${{ github.workspace }}/khronos-pkg/include
          ls -la "$LIB_DIR"
          echo "lib_dir=$LIB_DIR" >> "$GITHUB_OUTPUT"
          echo "include_dir=$INCLUDE_DIR" >> "$GITHUB_OUTPUT"

      - name: Clone openvx-mark
        run: |
          git clone --depth 1 https://github.com/kiritigowda/openvx-mark.git \
            ${{ github.workspace }}/openvx-mark

      # ---------------------------------------------------------------------
      # rustVX benchmark
      # ---------------------------------------------------------------------
      - name: Build openvx-mark against rustVX
        run: |
          mkdir -p ${{ github.workspace }}/openvx-mark/build-rustvx
          cd ${{ github.workspace }}/openvx-mark/build-rustvx
          cmake \
            -DCMAKE_BUILD_TYPE=Release \
            -DOPENVX_INCLUDES=${{ steps.rustvx.outputs.include_dir }} \
            -DOPENVX_LIB_DIR=${{ steps.rustvx.outputs.lib_dir }} \
            ..
          cmake --build . -j$(nproc)

      - name: Run benchmark (rustVX)
        run: |
          cd ${{ github.workspace }}/openvx-mark/build-rustvx
          export LD_LIBRARY_PATH=${{ steps.rustvx.outputs.lib_dir }}:$LD_LIBRARY_PATH
          ./openvx-mark --resolution VGA --iterations 10 --warmup 3

      # ---------------------------------------------------------------------
      # Khronos sample benchmark
      # ---------------------------------------------------------------------
      - name: Build openvx-mark against Khronos sample
        run: |
          mkdir -p ${{ github.workspace }}/openvx-mark/build-khronos
          cd ${{ github.workspace }}/openvx-mark/build-khronos
          cmake \
            -DCMAKE_BUILD_TYPE=Release \
            -DOPENVX_INCLUDES=${{ steps.khronos.outputs.include_dir }} \
            -DOPENVX_LIB_DIR=${{ steps.khronos.outputs.lib_dir }} \
            ..
          cmake --build . -j$(nproc)

      - name: Run benchmark (Khronos sample)
        run: |
          cd ${{ github.workspace }}/openvx-mark/build-khronos
          export LD_LIBRARY_PATH=${{ steps.khronos.outputs.lib_dir }}:$LD_LIBRARY_PATH
          ./openvx-mark --resolution VGA --iterations 10 --warmup 3

      # ---------------------------------------------------------------------
      # Compare results
      # ---------------------------------------------------------------------
      - name: Compare benchmark results (rustVX vs Khronos)
        run: |
          RUSTVX=${{ github.workspace }}/openvx-mark/build-rustvx/benchmark_results/benchmark_results.json
          KHRONOS=${{ github.workspace }}/openvx-mark/build-khronos/benchmark_results/benchmark_results.json

          if [ ! -f "$RUSTVX" ] || [ ! -f "$KHRONOS" ]; then
            echo "Skipping comparison — one or both benchmark results missing"
            ls -la "$(dirname $RUSTVX)" 2>/dev/null || true
            ls -la "$(dirname $KHRONOS)" 2>/dev/null || true
            exit 0
          fi

          # `compare_reports.py` defines Speedup as
          #   speedup = throughput(report_b) / throughput(report_a)
          # i.e. ">1.00 means report_b is faster". To make the Speedup
          # column read as "rustVX over Khronos" (>1.00x = rustVX wins),
          # pass Khronos first (baseline / report_a) and rustVX second
          # (candidate / report_b).
          python3 ${{ github.workspace }}/openvx-mark/scripts/compare_reports.py \
            "$KHRONOS" "$RUSTVX" \
            --output ${{ github.workspace }}/openvx-mark/comparison

      - name: Post comparison to job summary
        if: always()
        run: |
          COMPARISON=${{ github.workspace }}/openvx-mark/comparison.md
          RUSTVX=${{ github.workspace }}/openvx-mark/build-rustvx/benchmark_results/benchmark_results.json
          KHRONOS=${{ github.workspace }}/openvx-mark/build-khronos/benchmark_results/benchmark_results.json

          # ----- Headline: aggregate speedup of rustVX over Khronos sample -----
          if [ -f "$RUSTVX" ] && [ -f "$KHRONOS" ]; then
            python3 - "$RUSTVX" "$KHRONOS" >> "$GITHUB_STEP_SUMMARY" <<'PY'
          import json, math, sys

          rustvx_path, khronos_path = sys.argv[1], sys.argv[2]
          with open(rustvx_path) as f: rustvx = json.load(f)
          with open(khronos_path) as f: khronos = json.load(f)

          def by_key(report):
              return {(r['name'], r['mode'], r['resolution']): r
                      for r in report.get('results', [])}

          a = by_key(rustvx)
          b = by_key(khronos)
          shared = sorted(set(a) & set(b))

          speedups = []
          wins, losses = 0, 0
          best = (None, 0.0)
          worst = (None, math.inf)

          for key in shared:
              ra, rb = a[key], b[key]
              if not (ra.get('verified', True) and rb.get('verified', True)):
                  continue
              mps_r = ra.get('megapixels_per_sec', 0)
              mps_k = rb.get('megapixels_per_sec', 0)
              if mps_r <= 0 or mps_k <= 0:
                  continue
              s = mps_r / mps_k  # >1.0 = rustVX faster than Khronos
              speedups.append(s)
              if s > 1.0: wins += 1
              elif s < 1.0: losses += 1
              if s > best[1]:  best  = (key, s)
              if s < worst[1]: worst = (key, s)

          print('# rustVX vs Khronos sample — headline')
          print()
          if not speedups:
              print('_No verified benchmarks were directly comparable._')
          else:
              geomean = math.exp(sum(math.log(s) for s in speedups) / len(speedups))
              median  = sorted(speedups)[len(speedups) // 2]
              print('| Metric | Value |')
              print('|:---|---:|')
              print(f'| Geomean speedup (rustVX / Khronos) | **{geomean:.2f}x** |')
              print(f'| Median speedup (rustVX / Khronos)  | {median:.2f}x |')
              print(f'| Benchmarks compared                | {len(speedups)} |')
              print(f'| rustVX faster                       | {wins} |')
              print(f'| Khronos sample faster               | {losses} |')
              if best[0]:
                  bk, bv = best
                  print(f'| Best  rustVX speedup                | {bv:.2f}x ({bk[0]} / {bk[1]} / {bk[2]}) |')
              if worst[0] and worst[1] != math.inf:
                  wk, wv = worst
                  print(f'| Worst rustVX speedup                | {wv:.2f}x ({wk[0]} / {wk[1]} / {wk[2]}) |')
              print()
              if geomean >= 1.0:
                  print(f'> rustVX is **{geomean:.2f}x** faster than the Khronos sample on average (geomean across {len(speedups)} verified benchmarks).')
              else:
                  print(f'> rustVX is **{1.0/geomean:.2f}x slower** than the Khronos sample on average (geomean across {len(speedups)} verified benchmarks).')
              print()
          PY
          fi

          # ----- Detailed comparison table from compare_reports.py -----
          if [ -f "$COMPARISON" ]; then
            cat "$COMPARISON" >> "$GITHUB_STEP_SUMMARY"
          else
            echo "_No comparison report was produced._" >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload rustVX benchmark results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results-rustvx
          path: ${{ github.workspace }}/openvx-mark/build-rustvx/benchmark_results/
          if-no-files-found: ignore

      - name: Upload Khronos sample benchmark results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results-khronos-sample
          path: ${{ github.workspace }}/openvx-mark/build-khronos/benchmark_results/
          if-no-files-found: ignore

      - name: Upload comparison report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-comparison
          path: ${{ github.workspace }}/openvx-mark/comparison.*
          if-no-files-found: ignore