BUILD/CI: Add ZE null-driver build and smoke check

buildlib/az-helpers.sh

@@ -97,6 +97,9 @@ function add_timestamp() {
 }
 
 function az_init_modules() {
+    # Public vendor images (e.g. the Intel oneAPI ZE builder) do not ship the
+    # Environment Modules system. Nothing to initialize there.
+    [ -f /etc/profile.d/modules.sh ] || return 0
     . /etc/profile.d/modules.sh
     export MODULEPATH="/hpc/local/etc/modulefiles:$MODULEPATH"
     # Read module files (W/A if there're some network instabilities lead to autofs issues)

buildlib/dockers/ze-builder.Dockerfile

@@ -0,0 +1,98 @@
+# UCX Level Zero (ZE) builder image.
+#
+# Local build:
+#   docker build -t ze-builder:local \
+#       -f buildlib/dockers/ze-builder.Dockerfile buildlib/dockers
+#
+# CI publish: this image is pulled by the ubuntu2404_ze container in
+# buildlib/pr/main.yml. Push it to the internal registry (not Docker Hub, whose
+# pull rate limits break CI) under a unique, content-reflecting tag, e.g.
+#   rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu24.04/builder:ze-oneapi-2026.0.0-l0-1.27.0
+# and keep main.yml's image ref in sync with the published tag.
+#
+# The Intel oneAPI toolkit base already ships the Level Zero loader, headers and
+# pkg-config, which is what UCX needs to configure --with-ze.
+FROM intel/oneapi-toolkit:2026.0.0-devel-ubuntu24.04
+
+# The null driver is NOT shipped by any release/runtime package — it only builds
+# with BUILD_L0_LOADER_TESTS=1 + INSTALL_NULL_DRIVER=1. We build it from the
+# level-zero source tag that MATCHES the loader already present in this image,
+# then install ONLY libze_null.so.* so the image's loader stays authoritative.
+# The match matters: the loader dlopens "libze_null.so.<loader_version>"
+# (MAKE_LIBRARY_NAME), so a version-mismatched null driver would not load.
+#
+# Tag of the level-zero source to build the null driver from. Must match the
+# loader version the base image ships (verified: intel/oneapi-toolkit:2026.0.0
+# ships loader 1.27.0). If set empty (ARG L0_TAG=), the build auto-detects the
+# tag from the image's libze_loader.so.1 soname instead of using this default.
+# Override: docker build --build-arg L0_TAG=v1.30.0
+ARG L0_TAG=v1.27.0
+
+# Build deps for level-zero (cmake/ninja/git) + UCX (autotools, IB headers).
+# Base image already provides a C/C++ toolchain.
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        autoconf \
+        automake \
+        cmake \
+        debhelper \
+        doxygen \
+        dpkg-dev \
+        fakeroot \
+        git \
+        libibverbs-dev \
+        librdmacm-dev \
+        libtool \
+        m4 \
+        make \
+        ninja-build \
+        pkg-config && \
+    rm -f /etc/ssh/ssh_host_* && \
+    rm -rf /var/lib/apt/lists/*
+
+# Build the in-tree null driver at the image loader's version and install ONLY it.
+RUN set -eux; \
+    # 1. Locate the versioned loader the image ships. Prefer the ldconfig cache,
+    #    then fall back to a filesystem search (oneAPI may install outside the
+    #    default linker path).
+    loader_real="$(ldconfig -p | awk '/libze_loader\.so\.1 /{print $NF; exit}')"; \
+    loader_real="$([ -n "${loader_real}" ] && readlink -f "${loader_real}" || true)"; \
+    case "${loader_real}" in \
+        *libze_loader.so.[0-9]*.[0-9]*.[0-9]*) ;; \
+        *) loader_real="$(find / -xdev -name 'libze_loader.so.[0-9]*.[0-9]*.[0-9]*' 2>/dev/null | head -n1)" ;; \
+    esac; \
+    test -n "${loader_real}" || { echo "ERROR: image has no versioned libze_loader.so"; exit 1; }; \
+    img_ver="$(printf '%s' "${loader_real}" | sed -nE 's/.*libze_loader\.so\.([0-9]+\.[0-9]+\.[0-9]+)$/\1/p')"; \
+    echo "Image loader: ${loader_real} (version ${img_ver:-unknown})"; \
+    # 2. Pick the source tag: explicit ARG wins, else the detected vX.Y.Z.
+    tag="${L0_TAG:-${img_ver:+v${img_ver}}}"; \
+    test -n "${tag}" || { echo "ERROR: could not detect loader version; pass --build-arg L0_TAG=vX.Y.Z"; exit 1; }; \
+    echo "Building null driver from level-zero ${tag}"; \
+    # 3. Clone that exact tag (fail loudly if the tag does not exist upstream).
+    git clone --depth 1 --branch "${tag}" \
+        https://github.com/oneapi-src/level-zero.git /tmp/level-zero; \
+    # 4. Configure with the test/null-driver targets enabled.
+    cmake -S /tmp/level-zero -B /tmp/level-zero/build -G Ninja \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DBUILD_L0_LOADER_TESTS=1 \
+        -DINSTALL_NULL_DRIVER=1; \
+    # 5. Build ONLY the null driver (we do NOT run `install` — that would
+    #    overwrite the image loader). zello_world is built best-effort below.
+    cmake --build /tmp/level-zero/build --target ze_null; \
+    # 6. Install just the null driver shared object + symlinks onto the linker path.
+    cp -P /tmp/level-zero/build/lib/libze_null.so* /usr/local/lib/; \
+    ldconfig; \
+    # 7. Hard build-time gate: the versioned null-driver soname must be resolvable.
+    ldconfig -p | grep -F 'libze_null.so.1'; \
+    # 8. Best-effort smoke: build+run zello_world IF the sample target is present
+    #    in this tag. The authoritative load gate is ucx_info in builds.sh; here
+    #    we only fail the image build when a sample that DID build then misbehaves.
+    if cmake --build /tmp/level-zero/build --target zello_world 2>/dev/null \
+        && [ -x /tmp/level-zero/build/bin/zello_world ]; then \
+        ZE_ENABLE_NULL_DRIVER=1 ZE_ENABLE_LOADER_DEBUG_TRACE=1 \
+            /tmp/level-zero/build/bin/zello_world; \
+    else \
+        echo "zello_world sample not available for ${tag}; skipping in-image smoke"; \
+    fi; \
+    # 9. Clean up source tree (driver already installed).
+    rm -rf /tmp/level-zero

buildlib/pr/build_job.yml

@@ -102,6 +102,10 @@ jobs:
           tencentos44:
             CONTAINER: tencentos44
             build_mode: short
+          ubuntu2404_ze:
+            CONTAINER: ubuntu2404_ze
+            build_mode: ze
+            require_ze: yes
       # ICC/PGI matrix
       ${{ if contains(parameters.name, 'icc_pgi') }}:
         matrix:
@@ -153,3 +157,7 @@ jobs:
           BUILD_ID: "$(Build.BuildId)-$(Build.BuildNumber)"
           build_mode: $(build_mode)
           test_static: $(test_static)
+          require_ze: $(require_ze)
+          # Intentionally unset by the default no-GPU ZE CI lane. Set this as
+          # a pipeline variable for manual Intel GPU validation runs.
+          require_real_ze: $(require_real_ze)

buildlib/pr/main.yml

@@ -2,6 +2,9 @@ variables:
   DOCKER_OPT_VOLUMES: -v /hpc/local:/hpc/local -v /auto/sw_tools:/auto/sw_tools
   DOCKER_OPT_IB: --ulimit memlock=-1:-1 --device=/dev/infiniband/ --net=host
   DOCKER_OPT_GPU: --gpus all --device=/dev/gdrdrv --ipc=host $(DOCKER_OPT_IB)
+  # Set to e.g. "--device=/dev/dri --group-add render --group-add video"
+  # when running the ZE lane on Intel GPU hosts.
+  DOCKER_OPT_ZE_GPU: ""
   DOCKER_OPT_ARGS: --cap-add=SYS_PTRACE
 
 resources:
@@ -78,6 +81,11 @@ resources:
     - container: ubuntu2404_doca31_gpunetio
       image: rdmz-harbor.rdmz.labs.mlnx/hpcx/x86_64/ubuntu24.04/builder:doca-3.1.0-gpunetio
       options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(DOCKER_OPT_GPU)
+    # No-GPU Level Zero builder (ZE loader + null driver), built from
+    # buildlib/dockers/ze-builder.Dockerfile.
+    - container: ubuntu2404_ze
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/x86_64/ubuntu24.04/builder:ze-oneapi-2026.0.0-l0-1.27.0
+      options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_ZE_GPU)
     - container: ubuntu2604
       image: rdmz-harbor.rdmz.labs.mlnx/hpcx/x86_64/ubuntu26.04/builder:mofed-26.04-0.7.6.0
       options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)

buildlib/tools/builds.sh

@@ -17,8 +17,19 @@ build_mode=${build_mode:-}
 
 build_mode=${build_mode:-long}
 
+require_ze=${require_ze:-}
+# Azure passes the literal template variable when a matrix row does not define
+# require_ze. Treat it as unset.
+[ "${require_ze}" = "\$(require_ze)" ] && require_ze=
+
+require_real_ze=${require_real_ze:-}
+# Azure passes the literal template variable when a matrix row does not define
+# require_real_ze. Treat it as unset so default no-GPU CI can use the ZE null
+# driver, while manual Intel GPU runs can set require_real_ze=yes.
+[ "${require_real_ze}" = "\$(require_real_ze)" ] && require_real_ze=
+
 case "${build_mode}" in
-long|short|sanity|compilers|soname_suffix)
+long|short|sanity|compilers|soname_suffix|ze)
 	;;
 *)
 	azure_log_error "Unsupported build mode: ${build_mode}"
@@ -330,6 +341,79 @@ build_rocm() {
 	fi
 }
 
+#
+# Confirm the ZE memory domain and transport enumerate in ucx_info output.
+#
+check_ze_devices() {
+	local ze_info=$1
+	local mode=$2
+
+	echo "${ze_info}"
+	if ! echo "${ze_info}" | grep -q "ze_cpy"; then
+		azure_log_error "ZE ${mode} smoke failed: ze_cpy MD not found"
+		exit 1
+	fi
+	if ! echo "${ze_info}" | grep -q "ze_copy"; then
+		azure_log_error "ZE ${mode} smoke failed: ze_copy transport not found"
+		exit 1
+	fi
+}
+
+#
+# Build with Intel Level Zero (ZE) and run a smoke check.
+#
+# Like build_cuda/build_rocm, this detects real hardware at runtime: if a real
+# Intel GPU is present (ZE enumerates a device without the null driver), it runs
+# the real smoke; otherwise it falls back to the in-tree Level Zero "null
+# driver", which returns synthetic success for init/discovery/object-lifecycle
+# calls. This lets the same function run real on an Intel GPU host or cluster
+# and no-GPU in the public Intel oneAPI CI container.
+#
+build_ze() {
+	if [ "${require_ze}" != "yes" ]; then
+		echo "==== Not building with ZE (require_ze!=yes) ===="
+		return
+	fi
+
+	echo "==== Build with enable ZE ===="
+	${WORKSPACE}/contrib/configure-devel --prefix=$ucx_inst \
+		"${compile_only_config_args[@]}" --with-ze
+	$MAKEP
+
+	# Compilation check: configure must have detected ZE.
+	grep '#define HAVE_ZE 1' config.h
+
+	$MAKEP install
+
+	# Probe for a real Intel GPU: query ZE without the null driver. If the
+	# ze_copy transport enumerates a device, real hardware is present.
+	local real_ze_info
+	real_ze_info=$(${ucx_inst}/bin/ucx_info -d 2>/dev/null || true)
+	if echo "${real_ze_info}" | grep -q "ze_copy"; then
+		echo "==== Running ZE smoke on real Intel GPU ===="
+		check_ze_devices "${real_ze_info}" "real-GPU"
+	else
+		if [ "${require_real_ze}" = "yes" ]; then
+			echo "${real_ze_info}"
+			azure_log_error "ZE real-GPU smoke failed: ze_copy transport not found"
+			exit 1
+		fi
+
+		# No real GPU: load the Level Zero null driver (synthetic device).
+		# ZE_ENABLE_NULL_DRIVER must be the literal "1"; the loader's
+		# getenv_tobool rejects anything else. The null driver soname is
+		# resolved from the linker cache (ldconfig in the builder image),
+		# so no ZEL_LIBRARY_PATH is needed.
+		echo "==== Running ZE null-driver smoke (no GPU) ===="
+		local null_ze_info
+		null_ze_info=$(ZE_ENABLE_NULL_DRIVER=1 ZE_ENABLE_LOADER_DEBUG_TRACE=1 \
+			${ucx_inst}/bin/ucx_info -d)
+		check_ze_devices "${null_ze_info}" "null-driver"
+	fi
+
+	make_clean distclean
+}
+
 #
 # Build with clang compiler
 #
@@ -548,6 +632,9 @@ compilers)
 soname_suffix)
 	tests=('build_soname_suffix')
 	;;
+ze)
+	tests=('build_ze')
+	;;
 esac
 
 num_tests=${#tests[@]}