opencv-mark: OpenCV baseline companion binary + CI integration #61

Workflow file for this run

	name: CI

	on:
	push:
	branches: [main]
	# Run CI on pull requests targeting any base branch, not just main.
	# This keeps stacked PR workflows covered (a PR's base may be another
	# feature branch, e.g. an umbrella branch or a previous PR in a stack).
	pull_request:

	# Auto-cancel superseded runs on the same ref so a rapid push series
	# (e.g. force-push during PR review) doesn't queue 3+ stale runs and
	# starve the GitHub Actions runner pool. main pushes are exempt — we
	# always want a clean signal on main.
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

	# ============================================================================
	# Architecture
	#
	# Phase 1 (parallel) — four independent build jobs:
	# * Three OpenVX-impl jobs (MIVisionX, Khronos sample, rustVX). Each:
	# 1. Builds the implementation from source.
	# 2. Stages a self-contained artifact: <impl>-stage/lib + <impl>-stage/include.
	# 3. Builds openvx-mark against the just-built impl.
	# 4. Runs a quick smoke benchmark (`--feature-set everything --resolution VGA`)
	# as a "local unit test" — catches build-link breakage and missing-symbol
	# issues immediately, scoped to the specific impl, without waiting for
	# the slower comparison job downstream.
	# 5. Uploads the staged artifact for the comparison job to consume.
	# * One OpenCV-baseline job (opencv-mark companion binary). Differs from
	# the OpenVX jobs because OpenCV is apt-installable and opencv-mark has
	# no OpenVX dependency — see the build-opencv job below for the shape.
	# Stages its smoke JSON directly (no impl tarball needed).
	#
	# Phase 2 (single job, depends on all four Phase-1 jobs) — comparison.
	# 1. Downloads all three OpenVX impl artifacts onto a single runner;
	# apt-installs OpenCV on that same runner.
	# 2. Builds openvx-mark × 3 (one per OpenVX impl) so all binaries link
	# against the same openvx-mark source tree at the same commit.
	# Builds opencv-mark from the same source tree.
	# 3. Runs the full benchmark (`--feature-set vision[,framework]
	# --resolution FHD`) against each impl. Same hardware = fair
	# cross-vendor comparison. opencv-mark runs `vision` only — it
	# deliberately doesn't implement the OpenVX `framework` feature
	# set, and `compare_reports.py` ignores not-on-both-sides kernels.
	# 4. Generates six pairwise comparison reports:
	# OpenVX-vs-OpenVX:
	# * MIVisionX vs Khronos sample
	# * MIVisionX vs rustVX
	# * Khronos sample vs rustVX
	# OpenVX-vs-OpenCV (the "does adopting OpenVX pay off?" trio):
	# * MIVisionX vs OpenCV
	# * Khronos sample vs OpenCV
	# * rustVX vs OpenCV
	# 5. Posts each report to the job summary and uploads as an artifact.
	#
	# Inspired by the layered build/perf-gate design in rustVX's conformance CI:
	# https://github.com/kiritigowda/rustVX/blob/main/.github/workflows/conformance.yml
	# ============================================================================

	jobs:
	# --------------------------------------------------------------------------
	# Phase 1 — MIVisionX (AMD OpenVX, CPU backend)
	# --------------------------------------------------------------------------
	build-mivisionx:
	name: Build MIVisionX (CPU) + smoke test
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout openvx-mark
	uses: actions/checkout@v4

	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y build-essential cmake git python3

	- name: Build MIVisionX (CPU backend)
	run: \|
	set -euo pipefail
	git clone --depth 1 --branch develop \
	https://github.com/ROCm/MIVisionX.git /tmp/mivisionx-src
	mkdir -p /tmp/mivisionx-src/build
	cd /tmp/mivisionx-src/build
	cmake \
	-DBACKEND=CPU \
	-DNEURAL_NET=OFF \
	-DLOOM=OFF \
	-DMIGRAPHX=OFF \
	-DCMAKE_INSTALL_PREFIX=/tmp/mivisionx-install \
	..
	make -j$(nproc)
	make install

	- name: Stage MIVisionX artifact
	id: stage
	run: \|
	set -euo pipefail
	mkdir -p mivisionx-stage/lib mivisionx-stage/include
	LIB_SRC=$(dirname "$(find /tmp/mivisionx-install -name 'libopenvx.so' \| head -1)")
	echo "MIVisionX libraries discovered in: $LIB_SRC"
	# Copy ALL libopenvx* / libvxu* entries (libopenvx.so symlink,
	# libopenvx.so.1 SONAME symlink, libopenvx.so.X.Y.Z real file)
	# preserving symlinks (-P) so ld.so can follow the SONAME chain.
	# Without versioned files the linker reports
	# "libopenvx.so.1: cannot open shared object file".
	find "$LIB_SRC" -maxdepth 1 -name 'libopenvx*' -exec cp -P {} mivisionx-stage/lib/ \;
	find "$LIB_SRC" -maxdepth 1 -name 'libvxu*' -exec cp -P {} mivisionx-stage/lib/ \;
	cp -r /tmp/mivisionx-install/include/mivisionx/. mivisionx-stage/include/
	echo "--- staged lib ---"
	ls -la mivisionx-stage/lib
	echo "--- staged include (top-level) ---"
	ls -la mivisionx-stage/include
	{
	echo "lib_dir=$(pwd)/mivisionx-stage/lib"
	echo "include_dir=$(pwd)/mivisionx-stage/include"
	} >> "$GITHUB_OUTPUT"

	- name: Build openvx-mark (smoke)
	run: \|
	set -euo pipefail
	mkdir -p build-smoke
	cd build-smoke
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ steps.stage.outputs.include_dir }} \
	-DOPENVX_LIB_DIR=${{ steps.stage.outputs.lib_dir }} \
	..
	cmake --build . -j$(nproc)

	# Smoke covers the `vision` + `framework` feature sets — i.e.
	# the comparison-stable subset that all three impls support
	# uniformly. `enhanced_vision` is excluded because at least one
	# impl (Khronos sample) segfaults inside Tensor* kernels, and
	# since openvx-mark only writes its JSON at end-of-run a mid-bench
	# crash takes out the entire signal. Re-enable per-impl gating
	# later when those impl quirks are sorted upstream.
	- name: Run smoke benchmark (vision + framework, VGA × 5 iters)
	# Smoke is advisory — if a specific impl crashes inside a
	# specific kernel the artifact upload (which the compare job
	# depends on) must still happen so vendor-vs-vendor signal
	# isn't lost.
	continue-on-error: true
	run: \|
	set -eo pipefail
	cd build-smoke
	export LD_LIBRARY_PATH=${{ steps.stage.outputs.lib_dir }}:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set vision,framework \
	--resolution VGA --iterations 5 --warmup 1 \
	--output-dir smoke-results

	- name: Upload MIVisionX artifact
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: impl-mivisionx
	path: mivisionx-stage/
	retention-days: 1

	- name: Upload MIVisionX smoke results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: smoke-results-mivisionx
	path: build-smoke/smoke-results/
	if-no-files-found: ignore

	# --------------------------------------------------------------------------
	# Phase 1 — Khronos OpenVX sample implementation
	# --------------------------------------------------------------------------
	build-khronos-sample:
	name: Build Khronos sample + smoke test
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout openvx-mark
	uses: actions/checkout@v4

	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y build-essential cmake git python3

	- name: Build Khronos OpenVX sample
	run: \|
	set -euo pipefail
	git clone --recursive --depth 1 \
	https://github.com/KhronosGroup/OpenVX-sample-impl.git /tmp/khronos-src
	cd /tmp/khronos-src
	python3 Build.py --os=Linux --arch=64 --conf=Release

	- name: Stage Khronos sample artifact
	id: stage
	run: \|
	set -euo pipefail
	mkdir -p khronos-stage/lib khronos-stage/include
	LIB_SRC=$(dirname "$(find /tmp/khronos-src -name 'libopenvx.so' -not -path '/build/' \| head -1)")
	echo "Khronos libraries discovered in: $LIB_SRC"
	# Same approach as MIVisionX: copy all libopenvx* / libvxu* entries
	# preserving symlinks so ld.so can follow the SONAME chain.
	find "$LIB_SRC" -maxdepth 1 -name 'libopenvx*' -exec cp -P {} khronos-stage/lib/ \;
	find "$LIB_SRC" -maxdepth 1 -name 'libvxu*' -exec cp -P {} khronos-stage/lib/ \;
	cp -r /tmp/khronos-src/api-docs/include/. khronos-stage/include/
	echo "--- staged lib ---"
	ls -la khronos-stage/lib
	echo "--- staged include (top-level) ---"
	ls -la khronos-stage/include
	{
	echo "lib_dir=$(pwd)/khronos-stage/lib"
	echo "include_dir=$(pwd)/khronos-stage/include"
	} >> "$GITHUB_OUTPUT"

	- name: Build openvx-mark (smoke)
	run: \|
	set -euo pipefail
	mkdir -p build-smoke
	cd build-smoke
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ steps.stage.outputs.include_dir }} \
	-DOPENVX_LIB_DIR=${{ steps.stage.outputs.lib_dir }} \
	..
	cmake --build . -j$(nproc)

	# Smoke is restricted to the `vision` + `framework` subset for
	# the same reason as the MIVisionX smoke — see comment there.
	- name: Run smoke benchmark (vision + framework, VGA × 5 iters)
	continue-on-error: true
	run: \|
	set -eo pipefail
	cd build-smoke
	export LD_LIBRARY_PATH=${{ steps.stage.outputs.lib_dir }}:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set vision,framework \
	--resolution VGA --iterations 5 --warmup 1 \
	--output-dir smoke-results

	- name: Upload Khronos sample artifact
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: impl-khronos-sample
	path: khronos-stage/
	retention-days: 1

	- name: Upload Khronos sample smoke results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: smoke-results-khronos-sample
	path: build-smoke/smoke-results/
	if-no-files-found: ignore

	# --------------------------------------------------------------------------
	# Phase 1 — rustVX (Rust OpenVX implementation)
	#
	# rustVX ships a single libopenvx_ffi.so that exports the full vx/vxu
	# symbol set. openvx-mark's CMake uses find_library(NAMES openvx) and
	# find_library(NAMES vxu) — so we symlink the two classic Khronos lib
	# names to the FFI .so during staging, without modifying rustVX's own
	# build output.
	#
	# SIMD config: AVX2 + `-C target-cpu=x86-64-v3`, matching what rustVX's
	# own CI ships. We deliberately skip the alignment-pad RUSTFLAGS used in
	# rustVX's PR-vs-main perf gate — those exist to make rustVX-vs-rustVX
	# bench numbers invariant to .text shifts, which is irrelevant for the
	# vendor-vs-vendor comparison this workflow runs.
	# --------------------------------------------------------------------------
	build-rustvx:
	name: Build rustVX + smoke test
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout openvx-mark
	uses: actions/checkout@v4

	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y build-essential cmake git

	- name: Install Rust toolchain
	run: \|
	set -euo pipefail
	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
	\| sh -s -- -y --default-toolchain stable
	source "$HOME/.cargo/env"
	rustc --version
	cargo --version

	- name: Build rustVX (release, AVX2)
	run: \|
	set -euo pipefail
	source "$HOME/.cargo/env"
	git clone --depth 1 \
	https://github.com/kiritigowda/rustVX.git /tmp/rustvx-src
	cd /tmp/rustvx-src
	case "$(uname -m)" in
	x86_64\|amd64)
	FEATURES="openvx-core/sse2 openvx-core/avx2 openvx-vision/sse2 openvx-vision/avx2"
	export RUSTFLAGS="-C target-cpu=x86-64-v3"
	;;
	aarch64\|arm64)
	FEATURES="openvx-core/neon openvx-vision/neon"
	export RUSTFLAGS=""
	;;
	*)
	FEATURES=""
	export RUSTFLAGS=""
	;;
	esac
	echo "Architecture : $(uname -m)"
	echo "Cargo features: ${FEATURES:-<none>}"
	echo "RUSTFLAGS : ${RUSTFLAGS:-<none>}"
	if [ -n "$FEATURES" ]; then
	cargo build --release -p openvx-ffi --features "$FEATURES"
	else
	cargo build --release -p openvx-ffi
	fi

	- name: Stage rustVX artifact (with libopenvx / libvxu symlinks)
	id: stage
	run: \|
	set -euo pipefail
	mkdir -p rustvx-stage/lib rustvx-stage/include
	cp /tmp/rustvx-src/target/release/libopenvx_ffi.so rustvx-stage/lib/
	# Classic Khronos library names so openvx-mark's find_library picks
	# them up. Symlinks survive upload-artifact@v4 (it preserves them
	# within tar), so the comparison job downstream sees the same.
	(
	cd rustvx-stage/lib
	ln -sf libopenvx_ffi.so libopenvx.so
	ln -sf libopenvx_ffi.so libvxu.so
	)
	cp -r /tmp/rustvx-src/include/. rustvx-stage/include/
	echo "--- staged lib ---"
	ls -la rustvx-stage/lib
	echo "--- staged include (top-level) ---"
	ls -la rustvx-stage/include
	{
	echo "lib_dir=$(pwd)/rustvx-stage/lib"
	echo "include_dir=$(pwd)/rustvx-stage/include"
	} >> "$GITHUB_OUTPUT"

	- name: Build openvx-mark (smoke)
	run: \|
	set -euo pipefail
	mkdir -p build-smoke
	cd build-smoke
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ steps.stage.outputs.include_dir }} \
	-DOPENVX_LIB_DIR=${{ steps.stage.outputs.lib_dir }} \
	..
	cmake --build . -j$(nproc)

	# Smoke is restricted to the `vision` + `framework` subset for
	# the same reason as the MIVisionX smoke — see comment there.
	- name: Run smoke benchmark (vision + framework, VGA × 5 iters)
	continue-on-error: true
	run: \|
	set -eo pipefail
	cd build-smoke
	export LD_LIBRARY_PATH=${{ steps.stage.outputs.lib_dir }}:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set vision,framework \
	--resolution VGA --iterations 5 --warmup 1 \
	--output-dir smoke-results

	- name: Upload rustVX artifact
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: impl-rustvx
	path: rustvx-stage/
	retention-days: 1

	- name: Upload rustVX smoke results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: smoke-results-rustvx
	path: build-smoke/smoke-results/
	if-no-files-found: ignore

	# --------------------------------------------------------------------------
	# Phase 1 — OpenCV baseline (companion binary `opencv-mark`)
	#
	# OpenCV is the de facto vision baseline. This job exists so we can answer
	# "does adopting OpenVX actually pay off vs the cv:: code I already have?"
	# at the per-kernel level, on the same CI hardware as every OpenVX impl.
	#
	# Differs from the OpenVX impl jobs in two ways:
	# 1. OpenCV is apt-installable (no from-source build), so this job is
	# much shorter — install, configure parent CMake, build, smoke.
	# 2. There is no impl-tarball staging step. opencv-mark IS the binary
	# that runs the OpenCV-side measurements; there is no separate
	# "link openvx-mark against this libopenvx.so" rebuild downstream.
	# The Phase 2 comparison job re-runs opencv-mark itself (after a
	# fresh apt-install of OpenCV) for strict same-runner fairness vs
	# the per-impl benches — see compare job's `Build & bench
	# opencv-mark` step.
	#
	# The smoke run here is fast feedback only (catches build/link breakage
	# in <1 min on every PR); the comparison-grade FHD × 20 iter benchmark
	# lives in Phase 2 alongside the OpenVX impl benches.
	# --------------------------------------------------------------------------
	build-opencv:
	name: Build opencv-mark (OpenCV baseline) + smoke test
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout openvx-mark
	uses: actions/checkout@v4

	- name: Install dependencies (OpenCV 4 from apt)
	run: \|
	sudo apt-get update
	sudo apt-get install -y build-essential cmake git python3 \
	libopencv-dev
	# Sanity-print the OpenCV version that pkg-config sees so
	# comparison reports later can be cross-referenced against
	# exactly this version string.
	pkg-config --modversion opencv4 \|\| true

	- name: Configure & build opencv-mark
	run: \|
	set -euo pipefail
	mkdir -p build-opencv
	cd build-opencv
	# Parent CMake auto-includes opencv-mark/ when OpenCV is found.
	# No OPENVX_* flags needed — opencv-mark has no OpenVX dep.
	cmake -DCMAKE_BUILD_TYPE=Release ..
	cmake --build . --target opencv-mark -j$(nproc)
	# Fail loudly if the binary somehow didn't get produced (e.g.
	# OpenCV detection silently no-op'd). This is the exact failure
	# mode that PR #1's first CI run was missing.
	test -x opencv-mark/opencv-mark \
	\|\| { echo "ERROR: opencv-mark binary not built — OpenCV likely not detected by CMake"; exit 1; }
	# `--help` doubles as a version probe — it prints the opencv-mark
	# version line and the linked OpenCV version up top. PR1's CLI
	# does not implement a dedicated `--version` flag yet.
	./opencv-mark/opencv-mark --help \| head -3

	# Smoke covers the PR1 sentinel kernel set. Same shape as the
	# OpenVX-impl smokes (VGA × 5 iters, 1 warmup) so timing noise
	# stays comparable. Not continue-on-error — opencv-mark has no
	# impl-side quirks to tolerate; if a kernel breaks here it's our
	# bug.
	- name: Run smoke benchmark (vision, VGA × 5 iters)
	run: \|
	set -eo pipefail
	cd build-opencv
	./opencv-mark/opencv-mark --feature-set vision \
	--resolution VGA --iterations 5 --warmup 1 \
	--output-dir smoke-results

	- name: Upload opencv-mark smoke results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: smoke-results-opencv
	path: build-opencv/smoke-results/
	if-no-files-found: ignore

	# --------------------------------------------------------------------------
	# Phase 2 — Pairwise comparison
	#
	# Pulls all three OpenVX implementation artifacts onto the same runner,
	# plus apt-installs OpenCV, so every benchmark is exercised on identical
	# hardware. Builds openvx-mark once per OpenVX impl (against this commit's
	# source tree, not pre-built artifacts — keeps the comparison binary
	# identical apart from the linked OpenVX lib), builds opencv-mark from
	# the same source tree, runs the full feature-set bench against each,
	# and emits six pairwise comparison reports:
	#
	# OpenVX-vs-OpenVX (3):
	# * MIVisionX over Khronos sample — AMD over reference
	# * MIVisionX over rustVX — AMD over Rust impl
	# * rustVX over Khronos sample — Rust impl over reference
	#
	# OpenVX-vs-OpenCV (3) — "does adopting OpenVX pay off?":
	# * MIVisionX over OpenCV — best-tuned OpenVX vs cv::
	# * Khronos sample over OpenCV — reference OpenVX vs cv::
	# * rustVX over OpenCV — Rust OpenVX vs cv::
	#
	# `if: always()` + per-download `continue-on-error` + per-bench
	# `if: always() && steps.detect...` so a single failed build still
	# surfaces the comparison signal for whichever other impls are
	# available, instead of losing all visibility.
	# --------------------------------------------------------------------------
	compare:
	name: Pairwise comparison (MIVisionX, Khronos, rustVX, OpenCV)
	runs-on: ubuntu-22.04
	needs:
	- build-mivisionx
	- build-khronos-sample
	- build-rustvx
	- build-opencv
	if: always()
	steps:
	- name: Checkout openvx-mark
	uses: actions/checkout@v4

	- name: Install dependencies
	run: \|
	sudo apt-get update
	# libopencv-dev is needed so the Phase 2 `Build & bench
	# opencv-mark` step can re-link opencv-mark on this runner.
	# Strictly same-hardware fairness vs the per-impl benches.
	sudo apt-get install -y build-essential cmake git python3 \
	libopencv-dev
	pkg-config --modversion opencv4 \|\| true

	- name: Download MIVisionX artifact
	uses: actions/download-artifact@v4
	with:
	name: impl-mivisionx
	path: ${{ github.workspace }}/impl/mivisionx
	continue-on-error: true

	- name: Download Khronos sample artifact
	uses: actions/download-artifact@v4
	with:
	name: impl-khronos-sample
	path: ${{ github.workspace }}/impl/khronos
	continue-on-error: true

	- name: Download rustVX artifact
	uses: actions/download-artifact@v4
	with:
	name: impl-rustvx
	path: ${{ github.workspace }}/impl/rustvx
	continue-on-error: true

	- name: Detect available implementations
	id: detect
	run: \|
	set -euo pipefail
	for impl in mivisionx khronos rustvx; do
	lib="${{ github.workspace }}/impl/$impl/lib/libopenvx.so"
	if [ -e "$lib" ]; then
	echo "$impl: AVAILABLE ($lib)"
	chmod -R u+rwX "${{ github.workspace }}/impl/$impl/lib"
	echo "${impl}=true" >> "$GITHUB_OUTPUT"
	else
	echo "$impl: MISSING (artifact download failed or build job did not produce it)"
	echo "${impl}=false" >> "$GITHUB_OUTPUT"
	fi
	done

	# ----- Per-impl build + benchmark (FHD, 20 iter, 5 warmup) -----
	#
	# Each per-impl bench uses `if: always() && steps.detect...` because
	# GitHub Actions treats any explicit `if:` without `always()` as
	# implicit `success()` — meaning a crash in MIVisionX bench would
	# skip the Khronos / rustVX bench steps entirely and we'd lose all
	# comparison signal. With `always()` the three benches stay
	# independent and the comparison job downstream handles whichever
	# JSON files actually got produced.
	#
	# Feature set is `vision,framework` (the comparison-stable subset
	# — see the smoke step comments for why `enhanced_vision` is held
	# back for now).
	- name: Build & bench against MIVisionX
	if: always() && steps.detect.outputs.mivisionx == 'true'
	run: \|
	set -euo pipefail
	mkdir -p build-mivisionx
	cd build-mivisionx
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ github.workspace }}/impl/mivisionx/include \
	-DOPENVX_LIB_DIR=${{ github.workspace }}/impl/mivisionx/lib \
	..
	cmake --build . -j$(nproc)
	export LD_LIBRARY_PATH=${{ github.workspace }}/impl/mivisionx/lib:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set vision,framework \
	--resolution FHD --iterations 20 --warmup 5 \
	--output-dir results

	- name: Build & bench against Khronos sample
	if: always() && steps.detect.outputs.khronos == 'true'
	run: \|
	set -euo pipefail
	mkdir -p build-khronos
	cd build-khronos
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ github.workspace }}/impl/khronos/include \
	-DOPENVX_LIB_DIR=${{ github.workspace }}/impl/khronos/lib \
	..
	cmake --build . -j$(nproc)
	export LD_LIBRARY_PATH=${{ github.workspace }}/impl/khronos/lib:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set vision,framework \
	--resolution FHD --iterations 20 --warmup 5 \
	--output-dir results

	- name: Build & bench against rustVX
	if: always() && steps.detect.outputs.rustvx == 'true'
	run: \|
	set -euo pipefail
	mkdir -p build-rustvx
	cd build-rustvx
	cmake \
	-DCMAKE_BUILD_TYPE=Release \
	-DOPENVX_INCLUDES=${{ github.workspace }}/impl/rustvx/include \
	-DOPENVX_LIB_DIR=${{ github.workspace }}/impl/rustvx/lib \
	..
	cmake --build . -j$(nproc)
	export LD_LIBRARY_PATH=${{ github.workspace }}/impl/rustvx/lib:${LD_LIBRARY_PATH:-}
	./openvx-mark --feature-set vision,framework \
	--resolution FHD --iterations 20 --warmup 5 \
	--output-dir results

	# opencv-mark has no OpenVX dependency, so no OPENVX_* flags and no
	# detect-step gate — it only needs `libopencv-dev` (already installed
	# above). Same FHD × 20 iter × 5 warmup shape as the OpenVX benches
	# so per-kernel speedups are directly comparable. Feature-set is
	# `vision` only — opencv-mark deliberately does not implement the
	# `framework` set (OpenCV has no equivalent of the OpenVX graph
	# framework concept), and `compare_reports.py` ignores
	# not-on-both-sides kernels so the framework rows simply don't
	# appear in OpenCV pairwise reports.
	- name: Build & bench opencv-mark
	if: always()
	id: bench_opencv
	run: \|
	set -euo pipefail
	mkdir -p build-opencv-bench
	cd build-opencv-bench
	cmake -DCMAKE_BUILD_TYPE=Release ..
	cmake --build . --target opencv-mark -j$(nproc)
	test -x opencv-mark/opencv-mark \
	\|\| { echo "ERROR: opencv-mark not built — OpenCV detection failed in compare job"; exit 1; }
	./opencv-mark/opencv-mark --feature-set vision \
	--resolution FHD --iterations 20 --warmup 5 \
	--output-dir results

	# ----- Pairwise comparisons -----
	#
	# Each comparison is oriented as "<candidate> over <baseline>" so
	# the speedup column reads as `candidate / baseline` (>1.00x =
	# candidate is faster). The orientation is deliberate:
	#
	# OpenVX-vs-OpenVX trio — "how much faster is the more-tuned
	# impl than the reference":
	# * MIVisionX over Khronos sample (AMD over reference)
	# * MIVisionX over rustVX (AMD over Rust impl)
	# * rustVX over Khronos sample (Rust impl over reference)
	#
	# OpenVX-vs-OpenCV trio — "does adopting OpenVX pay off vs cv::":
	# * MIVisionX over OpenCV
	# * Khronos sample over OpenCV
	# * rustVX over OpenCV
	#
	# Mechanically, `scripts/compare_reports.py` computes
	# speedup = throughput(arg2) / throughput(arg1)
	# so the candidate is passed as the SECOND positional arg.
	#
	# The step does two things:
	# 1. Runs `compare_reports.py` once per pair to produce a
	# per-kernel detail .md in comparisons/. These also become
	# the `benchmark-comparisons` artifact for downstream tools.
	# 2. Invokes `scripts/ci_pairwise_summary.py` once to render
	# an organized GitHub Step Summary — TL;DR speedup matrix
	# at top, two grouped headline tables, and the per-kernel
	# detail tables collapsed inside <details> blocks. See the
	# script docstring for the config schema; this used to be a
	# ~115-line bash + inline-Python block and rendered ~600
	# lines into the summary.
	- name: Pairwise comparisons
	if: always()
	run: \|
	set -euo pipefail
	mkdir -p comparisons

	# Per-impl JSON report paths (parallel arrays keyed by impl id).
	IDS=(mivisionx khronos rustvx opencv)
	PATHS=(
	"build-mivisionx/results/benchmark_results.json"
	"build-khronos/results/benchmark_results.json"
	"build-rustvx/results/benchmark_results.json"
	"build-opencv-bench/results/benchmark_results.json"
	)
	LABELS=(
	"MIVisionX (AMD OpenVX)"
	"Khronos sample"
	"rustVX"
	"OpenCV"
	)

	# The 6 pairs, "<candidate> <baseline>". Order matches the
	# rendered summary table order: OpenVX-vs-OpenCV (headline
	# question) first, then OpenVX-vs-OpenVX.
	PAIRS=(
	"mivisionx opencv"
	"khronos opencv"
	"rustvx opencv"
	"mivisionx khronos"
	"mivisionx rustvx"
	"rustvx khronos"
	)

	# Phase 1 — per-kernel detail .md per pair where both inputs
	# exist. Missing-input pairs are silently skipped here; the
	# summary script renders a friendly "_Detail missing_" note
	# for them inside the collapsed <details> block.
	path_of() {
	for i in "${!IDS[@]}"; do
	if [ "${IDS[$i]}" = "$1" ]; then echo "${PATHS[$i]}"; return; fi
	done
	}
	for pair in "${PAIRS[@]}"; do
	read -r CAND BASE <<< "$pair"
	CAND_PATH=$(path_of "$CAND")
	BASE_PATH=$(path_of "$BASE")
	OUT="comparisons/${CAND}-over-${BASE}"
	if [ -f "$CAND_PATH" ] && [ -f "$BASE_PATH" ]; then
	python3 scripts/compare_reports.py "$BASE_PATH" "$CAND_PATH" --output "$OUT"
	else
	echo "Skipping detail for ${CAND}-over-${BASE}: missing ${CAND_PATH} or ${BASE_PATH}"
	fi
	done

	# Phase 2 — render the organized step summary. The config
	# below is the only place pair-grouping & intent text lives;
	# the helper handles matrix rendering, headline tables, and
	# the collapsed <details> blocks.
	cat > /tmp/pairwise-config.json <<'JSON'
	{
	"reports": {
	"mivisionx": {"label": "MIVisionX (AMD OpenVX)", "path": "build-mivisionx/results/benchmark_results.json"},
	"khronos": {"label": "Khronos sample", "path": "build-khronos/results/benchmark_results.json"},
	"rustvx": {"label": "rustVX", "path": "build-rustvx/results/benchmark_results.json"},
	"opencv": {"label": "OpenCV", "path": "build-opencv-bench/results/benchmark_results.json"}
	},
	"groups": [
	{
	"title": "OpenVX-vs-OpenCV — does adopting OpenVX pay off vs cv::?",
	"intent": "Speedup reads as `<OpenVX impl> / OpenCV`. Values >1.00x mean adopting that OpenVX impl pays off vs writing the equivalent directly in OpenCV — the headline question this comparison phase exists to answer. Ordered most-tuned (MIVisionX) → reference (Khronos sample) → Rust impl (rustVX) so the table walks the realistic best→worst range of the trade-off.",
	"pairs": [["mivisionx", "opencv"], ["khronos", "opencv"], ["rustvx", "opencv"]]
	},
	{
	"title": "OpenVX-vs-OpenVX — cross-implementation",
	"intent": "Speedup reads as `<candidate> / <baseline>`. MIVisionX (AMD, most-tuned) compared against both reference impls, then rustVX vs Khronos sample (Rust impl over reference).",
	"pairs": [["mivisionx", "khronos"], ["mivisionx", "rustvx"], ["rustvx", "khronos"]]
	}
	],
	"detail_dir": "comparisons"
	}
	JSON
	python3 scripts/ci_pairwise_summary.py --config /tmp/pairwise-config.json \
	>> "$GITHUB_STEP_SUMMARY"

	echo "--- comparison artifacts ---"
	ls -la comparisons/ \|\| true

	- name: Upload per-impl benchmark results
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: benchmark-results
	path: \|
	build-mivisionx/results/
	build-khronos/results/
	build-rustvx/results/
	build-opencv-bench/results/
	if-no-files-found: ignore

	- name: Upload pairwise comparisons
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: benchmark-comparisons
	path: comparisons/
	if-no-files-found: ignore

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

opencv-mark: OpenCV baseline companion binary + CI integration #61

Workflow file

opencv-mark: OpenCV baseline companion binary + CI integration #61

Uh oh!

Workflow file for this run