TEST/PERF: Compare device perftest BW/Lat against the PR base

Alexey-Rivkin · Alexey-Rivkin · commit 76a075289a6b · 2026-06-08T00:19:05.000+03:00
The GPU device-API (GDA / gdaki) perftests run in CI but only ever ran once on the PR, with no comparison, so a bandwidth/latency regression went unnoticed (e.g. the ~34% GDA bandwidth drop in #11424). The OSU perf pipeline can't cover this: OSU is host-initiated and never enters the GPU-kernel-initiated path, and its perf nodes lack GDA hardware. Add a before/after check inside the existing GPU leg: build the PR base (HEAD^1 of the merge ref) and run test_types_ucp_device_cuda on both the base and head builds, interleaving the runs so both see the same node load, then compare per-test BW/Lat. A regression above the threshold (default 15%, looser than the OSU 5% since the GPU CI nodes are shared and noisier) fails the job. master / non-PR builds (no HEAD^2) just run once for coverage, unchanged.
diff --git a/buildlib/tools/compare_ucx_perftest.py b/buildlib/tools/compare_ucx_perftest.py
@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2026. ALL RIGHTS RESERVED.
+# See file LICENSE for terms.
+#
+# Compare ucx_perftest device-CUDA results between a base build and a head
+# build and flag bandwidth/latency regressions. Tolerates noisy shared CI
+# nodes by taking the median across repeated runs and using a loose threshold.
+#
+# Input files are raw stdout of `ucx_perftest -b <batch>` (one file per run).
+# A batch test prints a "+<name>----" header line followed by a "Final:" line
+# with the result columns (see src/tools/perf/perftest_run.c).
+
+import argparse
+import os
+import re
+import statistics
+import sys
+
+HEADER_RE = re.compile(r"^\+([A-Za-z0-9_]+)")
+FLOAT_RE = re.compile(r"[-+]?\d*\.?\d+")
+
+
+def read_test_names(config_path):
+    names = []
+    with open(config_path) as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            names.append(line.split()[0])
+    return names
+
+
+def _metric_from_floats(name, floats):
+    """Return (bandwidth_MBs, latency_us) for one result row.
+
+    Multi-thread final row: iters, lat(us), bw(MB/s), msgrate        (4 cols)
+    Single-thread final row: iters, lat_pctl, lat_mom, lat_total,
+                             bw_mom, bw_total, mr_mom, mr_total       (8 cols)
+    """
+    if len(floats) >= 8:
+        return floats[5], floats[3]
+    if len(floats) >= 4:
+        return floats[2], floats[1]
+    return None, None
+
+
+def parse_file(path, names):
+    """Return {test_name: value} using the metric implied by the name.
+
+    Each batch test prints a "+<name>----" header then a "Final:" result line.
+    """
+    name_set = set(names)
+    results = {}
+    current = None
+    if not os.path.isfile(path):
+        return results
+    with open(path) as f:
+        for line in f:
+            m = HEADER_RE.match(line)
+            if m and m.group(1) in name_set:
+                current = m.group(1)
+            elif current is not None and line.startswith("Final:"):
+                floats = [float(x) for x in FLOAT_RE.findall(line)]
+                bw, lat = _metric_from_floats(current, floats)
+                val = bw if "_bw_" in current else lat
+                if val is not None:
+                    results[current] = val
+                current = None
+    return results
+
+
+def median_by_test(paths, names):
+    samples = {}
+    for p in paths:
+        for name, val in parse_file(p, names).items():
+            samples.setdefault(name, []).append(val)
+    return {name: statistics.median(vals) for name, vals in samples.items()}
+
+
+def regression_pct(name, base, head):
+    """Positive value == head is worse than base."""
+    if base == 0:
+        return 0.0
+    if "_bw_" in name:          # bandwidth: higher is better
+        return (base - head) / base * 100.0
+    return (head - base) / base * 100.0   # latency: lower is better
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--names", required=True,
+                    help="batch config file (test_types_ucp_device_cuda)")
+    ap.add_argument("--threshold", type=float, default=15.0,
+                    help="max tolerated regression %% (default 15)")
+    ap.add_argument("--base", nargs="+", required=True)
+    ap.add_argument("--head", nargs="+", required=True)
+    args = ap.parse_args()
+
+    names = read_test_names(args.names)
+    base = median_by_test(args.base, names)
+    head = median_by_test(args.head, names)
+
+    if not base or not head:
+        print("ERROR: no parseable result files (base=%d, head=%d). "
+              "Did the perftest runs produce output?" % (len(base), len(head)),
+              file=sys.stderr)
+        return 1
+
+    regressed = []
+    print("%-34s %12s %12s %9s" % ("test", "base", "head", "regr%"))
+    for name in names:
+        b = base.get(name)
+        h = head.get(name)
+        if b is not None and h is None:
+            # Ran on base but produced no result on head - a hang/crash is the
+            # loudest regression, so fail rather than silently skip.
+            print("%-34s %12.2f %12s   MISSING ON HEAD <== REGRESSION" %
+                  (name, b, "-"))
+            regressed.append((name, float("inf")))
+            continue
+        if b is None or h is None:
+            print("%-34s %12s %12s   (no baseline)" %
+                  (name, "-" if b is None else b, "-" if h is None else h))
+            continue
+        pct = regression_pct(name, b, h)
+        flag = " <== REGRESSION" if pct > args.threshold else ""
+        print("%-34s %12.2f %12.2f %8.1f%%%s" % (name, b, h, pct, flag))
+        if pct > args.threshold:
+            regressed.append((name, pct))
+
+    if regressed:
+        msg = "device perftest regression > %.0f%%: %s" % (
+            args.threshold,
+            ", ".join("%s %.0f%%" % (n, p) for n, p in regressed))
+        # Loud, build-failing error (Azure annotation + non-zero exit).
+        print("##vso[task.logissue type=error]" + msg)
+        print("FAIL: " + msg, file=sys.stderr)
+        return 1
+
+    print("No device perftest regression above %.0f%% threshold." %
+          args.threshold)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/contrib/test_jenkins.sh b/contrib/test_jenkins.sh
@@ -29,6 +29,10 @@ source $(dirname $0)/../buildlib/tools/common.sh
 WORKSPACE=${WORKSPACE:=$PWD}
 ucx_inst=${WORKSPACE}/install
 
+# Absolute path to the source tree, captured before any 'cd' so it stays valid
+# regardless of the current directory at call time.
+ucx_src_dir=$(cd "$(dirname "$0")" && pwd)
+
 if [ -z "$BUILD_NUMBER" ]; then
 	echo "Running interactive"
 	BUILD_NUMBER=1
@@ -178,6 +182,7 @@ run_client_server_app() {
 	server_addr_arg=$3
 	kill_server=$4
 	error_emulation=$5
+	out_file=${6:-}
 
 	server_port_arg="-p $server_port"
 	step_server_port
@@ -195,11 +200,19 @@ run_client_server_app() {
 		set +Ee
 	fi
 
-	taskset -c $affinity_client ${test_exe} ${test_args} ${server_addr_arg} ${server_port_arg} &
+	if [ -n "${out_file}" ]
+	then
+		taskset -c $affinity_client ${test_exe} ${test_args} ${server_addr_arg} ${server_port_arg} > "${out_file}" 2>&1 &
+	else
+		taskset -c $affinity_client ${test_exe} ${test_args} ${server_addr_arg} ${server_port_arg} &
+	fi
 	client_pid=$!
 
 	wait ${client_pid}
 
+	# Echo captured output so it still appears in the CI log.
+	[ -n "${out_file}" ] && cat "${out_file}" || true
+
 	if [ $error_emulation -eq 1 ]
 	then
 		set -eE
@@ -634,7 +647,53 @@ run_ucx_perftest_with_daemon() {
 }
 
 #
-# Run UCX performance cuda device test
+# Run the cuda device perftest against a given install prefix, capturing the
+# client output to a results file (when given).
+#
+run_device_perftest_at() {
+	local inst=$1
+	local out_file=${2:-}
+	# TODO: Run on all GPUs & NICs combinations
+	local ucp_test_args="-b ${inst}/share/ucx/perftest/test_types_ucp_device_cuda"
+	local ucp_client_args="-a cuda:0 $(hostname)"
+
+	# TODO: Run with cuda_ipc_tls (cuda_copy,rc,cuda_ipc)
+	export UCX_TLS="cuda_copy,rc,rc_gda"
+	run_client_server_app "${inst}/bin/ucx_perftest" "${ucp_test_args}" \
+		"${ucp_client_args}" 0 0 "${out_file}"
+	unset UCX_TLS
+}
+
+#
+# Build UCX at a specific commit into a separate prefix ($base_inst), used to
+# produce the "before" binaries for perf regression compare. Built in devel
+# mode to match the head leg's devel build (line ~1335); --enable-gtest is
+# omitted on purpose - it adds the gtest suite but does not change the
+# ucx_perftest / libucp / libuct codegen, so the comparison stays fair.
+#
+build_ucx_at_commit() {
+	local sha=$1
+	local base_src="${WORKSPACE}/base-src"
+	base_inst="${WORKSPACE}/install-base"
+
+	(cd "${WORKSPACE}" && git worktree add -f --detach "${base_src}" "${sha}") \
+		|| return 1
+
+	local rc=0
+	(
+		WORKSPACE="${base_src}"
+		ucx_inst="${base_inst}"
+		prepare
+		build devel --without-valgrind
+	) || rc=1
+
+	(cd "${WORKSPACE}" && git worktree remove --force "${base_src}") || true
+	return ${rc}
+}
+
+#
+# Run UCX performance cuda device test, and (on a PR build) compare the
+# bandwidth/latency against the PR base to catch device-API regressions.
 #
 run_ucx_perftest_cuda_device() {
 	if [ "X$have_cuda" == "Xno" ]; then
@@ -652,23 +711,50 @@ run_ucx_perftest_cuda_device() {
 		return 0
 	fi
 
-    echo "==== Running ucx_perftest with cuda kernel ===="
-	ucx_inst_ptest=$ucx_inst/share/ucx/perftest
-	ucx_perftest="$ucx_inst/bin/ucx_perftest"
-	ucp_test_args="-b $ucx_inst_ptest/test_types_ucp_device_cuda"
+	echo "==== Running ucx_perftest with cuda kernel ===="
+	local repeat="${UCX_PERFTEST_REPEAT:-3}"
+	# OSU perf (dedicated nodes) uses 5%; device perf runs on shared, noisier
+	# GPU CI nodes, so the default tolerance is higher.
+	local threshold="${UCX_PERFTEST_REGRESSION_THRESHOLD:-15}"
+	local res_dir="${WORKSPACE}/device_perf"
+	rm -rf "${res_dir}"
+	mkdir -p "${res_dir}"
+
+	# On a PR build the checkout is the merge ref: HEAD^1 is the target branch
+	# tip (base) and HEAD^2 is the PR head. master / non-merge builds have no
+	# HEAD^2, so there is nothing to compare against - run once for coverage.
+	if ! (cd "${WORKSPACE}" && git rev-parse --verify -q HEAD^2 >/dev/null 2>&1)
+	then
+		echo "==== Not a PR merge build; running device perftest without comparison ===="
+		run_device_perftest_at "${ucx_inst}" ""
+		return 0
+	fi
 
-	# TODO: Run on all GPUs & NICs combinations
-	ucp_client_args="-a cuda:0 $(hostname)"
-	gda_tls="cuda_copy,rc,rc_gda"
-	cuda_ipc_tls="cuda_copy,rc,cuda_ipc"
+	local base_sha
+	base_sha=$(cd "${WORKSPACE}" && git rev-parse HEAD^1)
+	echo "==== Building base ${base_sha} for device perftest comparison ===="
+	if ! build_ucx_at_commit "${base_sha}"
+	then
+		echo "==== Base build failed; running device perftest without comparison ===="
+		run_device_perftest_at "${ucx_inst}" ""
+		return 0
+	fi
 
-	# TODO: Run with cuda_ipc_tls
-	for tls in "$gda_tls"
+	# Interleave head/base runs so both see the same node-load window - on the
+	# shared GPU CI nodes, measuring them far apart would skew the comparison.
+	local i
+	for i in $(seq 1 "${repeat}")
 	do
-		export UCX_TLS=${tls}
-		run_client_server_app "$ucx_perftest" "$ucp_test_args" "$ucp_client_args" 0 0
+		run_device_perftest_at "${ucx_inst}"  "${res_dir}/head.${i}.txt"
+		run_device_perftest_at "${base_inst}" "${res_dir}/base.${i}.txt"
 	done
-	unset UCX_TLS
+
+	echo "==== Comparing device perftest: base vs head ===="
+	python3 "${ucx_src_dir}/../buildlib/tools/compare_ucx_perftest.py" \
+		--names "${ucx_inst}/share/ucx/perftest/test_types_ucp_device_cuda" \
+		--threshold "${threshold}" \
+		--base "${res_dir}"/base.*.txt \
+		--head "${res_dir}"/head.*.txt
 }
 
 #
