CI: parallel build matrix + pairwise vendor comparison (MIVisionX, Khronos, rustVX) #49

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main]
	# Run CI on pull requests targeting any base branch, not just main.
	# This keeps stacked PR workflows covered (a PR's base may be another
	# feature branch, e.g. an umbrella branch or a previous PR in a stack).
	pull_request:

	# Auto-cancel superseded runs on the same ref so a rapid push series
	# (e.g. force-push during PR review) doesn't queue 3+ stale runs and
	# starve the GitHub Actions runner pool. main pushes are exempt — we
	# always want a clean signal on main.
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

	# ============================================================================
	# Architecture
	#
	# Phase 1 (parallel) — three independent build jobs, one per OpenVX
	# implementation. Each job:
	# 1. Builds the implementation from source.
	# 2. Stages a self-contained artifact: <impl>-stage/lib + <impl>-stage/include.
	# 3. Builds openvx-mark against the just-built impl.
	# 4. Runs a quick smoke benchmark (`--feature-set everything --resolution VGA`)
	# as a "local unit test" — catches build-link breakage and missing-symbol
	# issues immediately, scoped to the specific impl, without waiting for the
	# slower comparison job downstream.
	# 5. Uploads the staged artifact for the comparison job to consume.
	#
	# Phase 2 (single job, depends on all three) — comparison.
	# 1. Downloads all three impl artifacts onto a single runner.
	# 2. Builds openvx-mark × 3 (one per impl) so all binaries link against the
	# same openvx-mark source tree at the same commit.
	# 3. Runs the full benchmark (`--feature-set everything --resolution FHD`)
	# against each impl. Same hardware = fair cross-vendor comparison.
	# 4. Generates three pairwise comparison reports:
	# * MIVisionX vs Khronos sample
	# * MIVisionX vs rustVX
	# * Khronos sample vs rustVX
	# 5. Posts each report to the job summary and uploads as an artifact.
	#
	# Inspired by the layered build/perf-gate design in rustVX's conformance CI:
	# https://github.com/kiritigowda/rustVX/blob/main/.github/workflows/conformance.yml
	# ============================================================================

	jobs:
	# --------------------------------------------------------------------------
	# Phase 1 — MIVisionX (AMD OpenVX, CPU backend)
	# --------------------------------------------------------------------------
	build-mivisionx:
	name: Build MIVisionX (CPU) + smoke test
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout openvx-mark
	uses: actions/checkout@v4

	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y build-essential cmake git python3

	- name: Build MIVisionX (CPU backend)
	run: \|
	set -euo pipefail
	git clone --depth 1 --branch develop \
	https://github.com/ROCm/MIVisionX.git /tmp/mivisionx-src
	mkdir -p /tmp/mivisionx-src/build
	cd /tmp/mivisionx-src/build
	cmake \
	-DBACKEND=CPU \
	-DNEURAL_NET=OFF \
	-DLOOM=OFF \
	-DMIGRAPHX=OFF \
	-DCMAKE_INSTALL_PREFIX=/tmp/mivisionx-install \
	..
	make -j$(nproc)
	make install

	- name: Stage MIVisionX artifact
	id: stage
	run: \|
	set -euo pipefail
	mkdir -p mivisionx-stage/lib mivisionx-stage/include
	LIB_SRC=$(dirname "$(find /tmp/mivisionx-install -name 'libopenvx.so' \| head -1)")
	echo "MIVisionX libraries discovered in: $LIB_SRC"
	cp "$LIB_SRC"/libopenvx*.so mivisionx-stage/lib/
	cp "$LIB_SRC"/libvxu*.so mivisionx-stage/lib/ 2>/dev/null \|\| true
	cp -r /tmp/mivisionx-install/include/mivisionx/. mivisionx-stage/include/
	echo "--- staged lib ---"
	ls -la mivisionx-stage/lib
	echo "--- staged include (top-level) ---"
	ls -la mivisionx-stage/include
	{
	echo "lib_dir=$(pwd)/mivisionx-stage/lib"
	echo "include_dir=$(pwd)/mivisionx-stage/include"
	} >> "$GITHUB_OUTPUT"

	- name: Build openvx-mark (smoke)
	run: \|
	set -euo pipefail
	mkdir -p build-smoke
	cd build-smoke
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ steps.stage.outputs.include_dir }} \
	-DOPENVX_LIB_DIR=${{ steps.stage.outputs.lib_dir }} \
	..
	cmake --build . -j$(nproc)

	- name: Run smoke benchmark (kernel + framework, VGA × 5 iters)
	run: \|
	set -euo pipefail
	cd build-smoke
	export LD_LIBRARY_PATH=${{ steps.stage.outputs.lib_dir }}:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set everything \
	--resolution VGA --iterations 5 --warmup 1 \
	--output-dir smoke-results

	- name: Upload MIVisionX artifact
	uses: actions/upload-artifact@v4
	with:
	name: impl-mivisionx
	path: mivisionx-stage/
	retention-days: 1

	- name: Upload MIVisionX smoke results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: smoke-results-mivisionx
	path: build-smoke/smoke-results/
	if-no-files-found: ignore

	# --------------------------------------------------------------------------
	# Phase 1 — Khronos OpenVX sample implementation
	# --------------------------------------------------------------------------
	build-khronos-sample:
	name: Build Khronos sample + smoke test
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout openvx-mark
	uses: actions/checkout@v4

	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y build-essential cmake git python3

	- name: Build Khronos OpenVX sample
	run: \|
	set -euo pipefail
	git clone --recursive --depth 1 \
	https://github.com/KhronosGroup/OpenVX-sample-impl.git /tmp/khronos-src
	cd /tmp/khronos-src
	python3 Build.py --os=Linux --arch=64 --conf=Release

	- name: Stage Khronos sample artifact
	id: stage
	run: \|
	set -euo pipefail
	mkdir -p khronos-stage/lib khronos-stage/include
	LIB_SRC=$(dirname "$(find /tmp/khronos-src -name 'libopenvx.so' -not -path '/build/' \| head -1)")
	echo "Khronos libraries discovered in: $LIB_SRC"
	cp "$LIB_SRC"/libopenvx*.so khronos-stage/lib/
	cp "$LIB_SRC"/libvxu*.so khronos-stage/lib/ 2>/dev/null \|\| true
	cp -r /tmp/khronos-src/api-docs/include/. khronos-stage/include/
	echo "--- staged lib ---"
	ls -la khronos-stage/lib
	echo "--- staged include (top-level) ---"
	ls -la khronos-stage/include
	{
	echo "lib_dir=$(pwd)/khronos-stage/lib"
	echo "include_dir=$(pwd)/khronos-stage/include"
	} >> "$GITHUB_OUTPUT"

	- name: Build openvx-mark (smoke)
	run: \|
	set -euo pipefail
	mkdir -p build-smoke
	cd build-smoke
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ steps.stage.outputs.include_dir }} \
	-DOPENVX_LIB_DIR=${{ steps.stage.outputs.lib_dir }} \
	..
	cmake --build . -j$(nproc)

	- name: Run smoke benchmark (kernel + framework, VGA × 5 iters)
	run: \|
	set -euo pipefail
	cd build-smoke
	export LD_LIBRARY_PATH=${{ steps.stage.outputs.lib_dir }}:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set everything \
	--resolution VGA --iterations 5 --warmup 1 \
	--output-dir smoke-results

	- name: Upload Khronos sample artifact
	uses: actions/upload-artifact@v4
	with:
	name: impl-khronos-sample
	path: khronos-stage/
	retention-days: 1

	- name: Upload Khronos sample smoke results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: smoke-results-khronos-sample
	path: build-smoke/smoke-results/
	if-no-files-found: ignore

	# --------------------------------------------------------------------------
	# Phase 1 — rustVX (Rust OpenVX implementation)
	#
	# rustVX ships a single libopenvx_ffi.so that exports the full vx/vxu
	# symbol set. openvx-mark's CMake uses find_library(NAMES openvx) and
	# find_library(NAMES vxu) — so we symlink the two classic Khronos lib
	# names to the FFI .so during staging, without modifying rustVX's own
	# build output.
	#
	# SIMD config: AVX2 + `-C target-cpu=x86-64-v3`, matching what rustVX's
	# own CI ships. We deliberately skip the alignment-pad RUSTFLAGS used in
	# rustVX's PR-vs-main perf gate — those exist to make rustVX-vs-rustVX
	# bench numbers invariant to .text shifts, which is irrelevant for the
	# vendor-vs-vendor comparison this workflow runs.
	# --------------------------------------------------------------------------
	build-rustvx:
	name: Build rustVX + smoke test
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout openvx-mark
	uses: actions/checkout@v4

	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y build-essential cmake git

	- name: Install Rust toolchain
	run: \|
	set -euo pipefail
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
	\| sh -s -- -y --default-toolchain stable
	source "$HOME/.cargo/env"
	rustc --version
	cargo --version

	- name: Build rustVX (release, AVX2)
	run: \|
	set -euo pipefail
	source "$HOME/.cargo/env"
	git clone --depth 1 \
	https://github.com/kiritigowda/rustVX.git /tmp/rustvx-src
	cd /tmp/rustvx-src
	case "$(uname -m)" in
	x86_64\|amd64)
	FEATURES="openvx-core/sse2 openvx-core/avx2 openvx-vision/sse2 openvx-vision/avx2"
	export RUSTFLAGS="-C target-cpu=x86-64-v3"
	;;
	aarch64\|arm64)
	FEATURES="openvx-core/neon openvx-vision/neon"
	export RUSTFLAGS=""
	;;
	*)
	FEATURES=""
	export RUSTFLAGS=""
	;;
	esac
	echo "Architecture : $(uname -m)"
	echo "Cargo features: ${FEATURES:-<none>}"
	echo "RUSTFLAGS : ${RUSTFLAGS:-<none>}"
	if [ -n "$FEATURES" ]; then
	cargo build --release -p openvx-ffi --features "$FEATURES"
	else
	cargo build --release -p openvx-ffi
	fi

	- name: Stage rustVX artifact (with libopenvx / libvxu symlinks)
	id: stage
	run: \|
	set -euo pipefail
	mkdir -p rustvx-stage/lib rustvx-stage/include
	cp /tmp/rustvx-src/target/release/libopenvx_ffi.so rustvx-stage/lib/
	# Classic Khronos library names so openvx-mark's find_library picks
	# them up. Symlinks survive upload-artifact@v4 (it preserves them
	# within tar), so the comparison job downstream sees the same.
	(
	cd rustvx-stage/lib
	ln -sf libopenvx_ffi.so libopenvx.so
	ln -sf libopenvx_ffi.so libvxu.so
	)
	cp -r /tmp/rustvx-src/include/. rustvx-stage/include/
	echo "--- staged lib ---"
	ls -la rustvx-stage/lib
	echo "--- staged include (top-level) ---"
	ls -la rustvx-stage/include
	{
	echo "lib_dir=$(pwd)/rustvx-stage/lib"
	echo "include_dir=$(pwd)/rustvx-stage/include"
	} >> "$GITHUB_OUTPUT"

	- name: Build openvx-mark (smoke)
	run: \|
	set -euo pipefail
	mkdir -p build-smoke
	cd build-smoke
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ steps.stage.outputs.include_dir }} \
	-DOPENVX_LIB_DIR=${{ steps.stage.outputs.lib_dir }} \
	..
	cmake --build . -j$(nproc)

	- name: Run smoke benchmark (kernel + framework, VGA × 5 iters)
	run: \|
	set -euo pipefail
	cd build-smoke
	export LD_LIBRARY_PATH=${{ steps.stage.outputs.lib_dir }}:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set everything \
	--resolution VGA --iterations 5 --warmup 1 \
	--output-dir smoke-results

	- name: Upload rustVX artifact
	uses: actions/upload-artifact@v4
	with:
	name: impl-rustvx
	path: rustvx-stage/
	retention-days: 1

	- name: Upload rustVX smoke results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: smoke-results-rustvx
	path: build-smoke/smoke-results/
	if-no-files-found: ignore

	# --------------------------------------------------------------------------
	# Phase 2 — Pairwise comparison
	#
	# Pulls all three implementation artifacts onto the same runner so every
	# benchmark is exercised on identical hardware. Builds openvx-mark once
	# per impl (against this commit's source tree, not pre-built artifacts —
	# keeps the comparison binary identical apart from the linked OpenVX lib),
	# runs the full feature-set bench against each, and emits three pairwise
	# comparison reports.
	#
	# `if: always()` + per-download `continue-on-error` so a single failed
	# build still surfaces the comparison signal for the other two impls
	# instead of losing all visibility.
	# --------------------------------------------------------------------------
	compare:
	name: Pairwise comparison (MIVisionX, Khronos, rustVX)
	runs-on: ubuntu-22.04
	needs:
	- build-mivisionx
	- build-khronos-sample
	- build-rustvx
	if: always()
	steps:
	- name: Checkout openvx-mark
	uses: actions/checkout@v4

	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y build-essential cmake git python3

	- name: Download MIVisionX artifact
	uses: actions/download-artifact@v4
	with:
	name: impl-mivisionx
	path: ${{ github.workspace }}/impl/mivisionx
	continue-on-error: true

	- name: Download Khronos sample artifact
	uses: actions/download-artifact@v4
	with:
	name: impl-khronos-sample
	path: ${{ github.workspace }}/impl/khronos
	continue-on-error: true

	- name: Download rustVX artifact
	uses: actions/download-artifact@v4
	with:
	name: impl-rustvx
	path: ${{ github.workspace }}/impl/rustvx
	continue-on-error: true

	- name: Detect available implementations
	id: detect
	run: \|
	set -euo pipefail
	for impl in mivisionx khronos rustvx; do
	lib="${{ github.workspace }}/impl/$impl/lib/libopenvx.so"
	if [ -e "$lib" ]; then
	echo "$impl: AVAILABLE ($lib)"
	chmod -R u+rwX "${{ github.workspace }}/impl/$impl/lib"
	echo "${impl}=true" >> "$GITHUB_OUTPUT"
	else
	echo "$impl: MISSING (artifact download failed or build job did not produce it)"
	echo "${impl}=false" >> "$GITHUB_OUTPUT"
	fi
	done

	# ----- Per-impl build + benchmark (FHD, 20 iter, 5 warmup) -----
	- name: Build & bench against MIVisionX
	if: steps.detect.outputs.mivisionx == 'true'
	run: \|
	set -euo pipefail
	mkdir -p build-mivisionx
	cd build-mivisionx
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ github.workspace }}/impl/mivisionx/include \
	-DOPENVX_LIB_DIR=${{ github.workspace }}/impl/mivisionx/lib \
	..
	cmake --build . -j$(nproc)
	export LD_LIBRARY_PATH=${{ github.workspace }}/impl/mivisionx/lib:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set everything \
	--resolution FHD --iterations 20 --warmup 5 \
	--output-dir results

	- name: Build & bench against Khronos sample
	if: steps.detect.outputs.khronos == 'true'
	run: \|
	set -euo pipefail
	mkdir -p build-khronos
	cd build-khronos
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ github.workspace }}/impl/khronos/include \
	-DOPENVX_LIB_DIR=${{ github.workspace }}/impl/khronos/lib \
	..
	cmake --build . -j$(nproc)
	export LD_LIBRARY_PATH=${{ github.workspace }}/impl/khronos/lib:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set everything \
	--resolution FHD --iterations 20 --warmup 5 \
	--output-dir results

	- name: Build & bench against rustVX
	if: steps.detect.outputs.rustvx == 'true'
	run: \|
	set -euo pipefail
	mkdir -p build-rustvx
	cd build-rustvx
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ github.workspace }}/impl/rustvx/include \
	-DOPENVX_LIB_DIR=${{ github.workspace }}/impl/rustvx/lib \
	..
	cmake --build . -j$(nproc)
	export LD_LIBRARY_PATH=${{ github.workspace }}/impl/rustvx/lib:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set everything \
	--resolution FHD --iterations 20 --warmup 5 \
	--output-dir results

	# ----- Pairwise comparisons -----
	#
	# `compare_reports.py` defines speedup as
	# speedup = throughput(report_b) / throughput(report_a)
	# i.e. ">1.00 = report_b is faster". We always pass the pair in the
	# same order as the report header label so the Speedup column reads
	# naturally as "<name_b> over <name_a>".
	- name: Pairwise comparisons
	if: always()
	run: \|
	set -euo pipefail
	mkdir -p comparisons

	M="build-mivisionx/results/benchmark_results.json"
	K="build-khronos/results/benchmark_results.json"
	R="build-rustvx/results/benchmark_results.json"

	do_compare() {
	local name_a="$1" name_b="$2" path_a="$3" path_b="$4"
	local out="comparisons/${name_a}-vs-${name_b}"
	{
	echo "## Comparison: ${name_a} vs ${name_b}"
	echo ""
	} >> "$GITHUB_STEP_SUMMARY"
	if [ ! -f "$path_a" ] \|\| [ ! -f "$path_b" ]; then
	echo "_Skipped: one or both reports missing (${name_a}: $([ -f "$path_a" ] && echo OK \|\| echo MISSING), ${name_b}: $([ -f "$path_b" ] && echo OK \|\| echo MISSING))_" \
	>> "$GITHUB_STEP_SUMMARY"
	echo "" >> "$GITHUB_STEP_SUMMARY"
	echo "---" >> "$GITHUB_STEP_SUMMARY"
	echo "" >> "$GITHUB_STEP_SUMMARY"
	return 0
	fi
	python3 scripts/compare_reports.py "$path_a" "$path_b" --output "$out"
	{
	echo "_Speedup column reads as ${name_b} / ${name_a} (>1.00x = ${name_b} faster)._"
	echo ""
	cat "${out}.md"
	echo ""
	echo "---"
	echo ""
	} >> "$GITHUB_STEP_SUMMARY"
	}

	do_compare mivisionx khronos "$M" "$K"
	do_compare mivisionx rustvx "$M" "$R"
	do_compare khronos rustvx "$K" "$R"

	echo "--- comparison artifacts ---"
	ls -la comparisons/ \|\| true

	- name: Upload per-impl benchmark results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: benchmark-results
	path: \|
	build-mivisionx/results/
	build-khronos/results/
	build-rustvx/results/
	if-no-files-found: ignore

	- name: Upload pairwise comparisons
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: benchmark-comparisons
	path: comparisons/
	if-no-files-found: ignore

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

CI: parallel build matrix + pairwise vendor comparison (MIVisionX, Khronos, rustVX) #49

Workflow file

CI: parallel build matrix + pairwise vendor comparison (MIVisionX, Khronos, rustVX) #49

Uh oh!

Workflow file for this run