DAOS-19005 test: Fix dfuse daos_build tests

src/tests/ftest/daos_build.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+#
+#  Copyright 2026 Hewlett Packard Enterprise Development LP
+#
+#  SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+# This is a script to be run by the src/tests/ftest/dfuse/daos_build.py to run a test on a CI node.
+
+set -euo pipefail
+
+timestamp_output() {
+    while IFS= read -r line; do
+        # Use bash printf time formatting to avoid spawning `date` per line
+        printf '%(%Y-%m-%d %H:%M:%S)T %s\n' -1 "$line"
+    done
+}
+
+# Timestamp all script output (stdout + stderr)
+exec > >(timestamp_output) 2>&1
+
+show_help() {
+    echo "Usage: $0 [options]"
+    echo "Options:"
+    echo "  -p, --python-cmd <val>    Python command to use (default: python3)"
+    echo "  -v, --venv-dir <val>      Path to Python virtual environment (default: /tmp/daos_build/venv)"
+    echo "  -b, --build-dir <val>     Directory to clone and build DAOS (default: /tmp/daos_build/daos)"
+    echo "  -m, --mount-dir <val>     Optional mount directory to use for filesystem tests (default: none)"
+    echo "  -g, --git-checkout <val>  Git branch or commit to checkout (default: origin/master)"
+    echo "  -d, --distro <val>        Linux distribution for installing dependencies (default: el9)"
+    echo "  -j, --build-jobs <val>    Number of parallel jobs for building DAOS (default: 30)"
+    echo "  -r, --rebuild             Whether to skip setup of build and venv directories (default: false)"
+    echo "      --debug               Whether to run additional debug checks (default: false)"
+    echo "  -h, --help                Show this help message and exit"
+}
+
+# Argument defaults
+python_cmd="python3"
+venv_dir="/tmp/daos_build/venv"
+build_dir="/tmp/daos_build/daos"
+mount_dir=""
+git_checkout="origin/master"
+distro="el9"
+build_jobs="30"
+rebuild="false"
+debug="false"
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -p|--python-cmd)
+      python_cmd="$2"
+      shift 2
+      ;;
+    -v|--venv-dir)
+      venv_dir="$2"
+      shift 2
+      ;;
+    -b|--build-dir)
+      build_dir="$2"
+      shift 2
+      ;;
+    -m|--mount-dir)
+      mount_dir="$2"
+      shift 2
+      ;;
+    -g|--git-checkout)
+      git_checkout="$2"
+      shift 2
+      ;;
+    -d|--distro)
+      distro="$2"
+      shift 2
+      ;;
+    -j|--build-jobs)
+      build_jobs="$2"
+      shift 2
+      ;;
+    -r|--rebuild)
+      rebuild="true"
+      shift 1
+      ;;
+    --debug)
+      debug="true"
+      shift 1
+      ;;
+    --help|-h)
+      show_help
+      exit 0
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      echo "Unknown option: $1" >&2
+      show_help
+      exit 1
+      ;;
+  esac
+done
+
+run_cmd() {
+    local start_time end_time elapsed_seconds rc
+    local cmd="$*"
+
+    echo ">---------------------------------------------------------"
+    echo "Running command: ${cmd}"
+    start_time=$(date +%s)
+    ${cmd}
+    rc=$?
+    end_time=$(date +%s)
+    elapsed_seconds=$((end_time - start_time))
+
+    echo "Command finished in ${elapsed_seconds}s with rc=${rc}: ${cmd}"
+    if [ "${rc}" -ne 0 ]; then
+        echo "ERROR: command failed with rc=${rc}: ${cmd}" # >&2
+        return "${rc}"
+    fi
+    return 0
+}
+
+full_start=$(date +%s)
+
+# Create a Python virtual environment and install python build dependencies
+if [ "${rebuild}" = "false" ]; then
+    run_cmd "rm -rf ${venv_dir}" || exit
+    run_cmd "${python_cmd} -m venv ${venv_dir}" || exit
+
+    cat <<EOF > "${venv_dir}"/pip.conf
+[global]
+    progress_bar = off
+    no_color = true
+    quiet = 0
+EOF
+fi
+
+# Run in the python virtual environment for the rest of the script
+run_cmd "source ${venv_dir}/bin/activate" || exit
+venv_python="${venv_dir}/bin/python"
+
+# Clone the DAOS repository and install RPM dependencies for the build
+if [ "${rebuild}" = "false" ]; then
+  run_cmd "rm -rf ${build_dir}" || exit
+  run_cmd "git clone https://github.com/daos-stack/daos.git ${build_dir}" || exit
+  run_cmd "git -C ${build_dir} checkout ${git_checkout}" || exit
+  run_cmd "git -C ${build_dir} submodule update --init --recursive" || exit
+
+  run_cmd "cp ${build_dir}/utils/scripts/install-${distro}.sh /tmp/install.sh" || exit
+  run_cmd "sudo -E NO_OPENMPI_DEVEL=1 /tmp/install.sh -y" || exit
+
+  run_cmd "${venv_python} -m pip install pip --upgrade" || exit
+  run_cmd "${venv_python} -m pip install -r ${build_dir}/requirements-build.txt" || exit
+fi
+
+# Debug
+if [ "${debug}" = "true" ]; then
+  # Check that all required Python packages are installed and available in PATH
+  awk '!/^\s*($|#)/ {print $1}' "${build_dir}/requirements-build.txt" | while read -r pkg; do
+    if [ "${pkg}" = "pyelftools" ]; then
+      continue
+    fi
+    run_cmd "which ${pkg}" || exit
+  done
+
+  run_cmd "which python" || true
+  run_cmd "which python3" || true
+  run_cmd "cat /etc/uv/uv.toml" || true
+  run_cmd "which uv" || true
+  run_cmd "echo $PATH" || true
+  run_cmd "echo $VIRTUAL_ENV" || true
+fi
+
+# Build DAOS dependencies
+scons="${venv_python} -m SCons"
+run_cmd "${scons} -C ${build_dir} --jobs ${build_jobs} --build-deps=only" || exit
+
+if [[ -n ${mount_dir-} ]]; then
+  # Run filesystem tests to verify the build.
+  run_cmd "daos filesystem query ${mount_dir}" || exit
+  run_cmd "daos filesystem evict ${build_dir}" || exit
+  run_cmd "daos filesystem query ${mount_dir}" || exit
+fi
+
+# Build and install DAOS
+run_cmd "${scons} -C ${build_dir} --jobs ${build_jobs}" || exit
+run_cmd "${scons} -C ${build_dir} --jobs ${build_jobs} install --implicit-deps-unchanged" || exit
+
+if [[ -n ${mount_dir-} ]]; then
+  run_cmd "daos filesystem query ${mount_dir}" || exit
+fi
+
+full_end=$(date +%s)
+full_time=$((full_end - full_start))
+echo "DAOS build and installation completed successfully in ${full_time}s"
+exit 0

src/tests/ftest/dfuse/daos_build.py

@@ -102,11 +102,10 @@ def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False):
     mount_dir = dfuse.mount_dir.value
     build_dir = os.path.join(mount_dir, 'daos')
 
-    remote_env['PATH'] = f"{os.path.join(mount_dir, 'venv', 'bin')}:$PATH"
-    remote_env['VIRTUAL_ENV'] = os.path.join(mount_dir, 'venv')
     remote_env['COVFILE'] = os.environ['COVFILE']
     remote_env['HTTPS_PROXY'] = os.environ.get('HTTPS_PROXY', '')
     remote_env['NO_PROXY'] = os.environ.get('NO_PROXY', '')
+    remote_env['SCONS_ENV'] = 'full'
 
     if il_lib is not None:
         remote_env['LD_PRELOAD'] = os.path.join(self.prefix, 'lib64', il_lib)
@@ -119,8 +118,6 @@ def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False):
             remote_env['D_IL_COMPATIBLE'] = '1'
             remote_env['D_IL_MAX_EQ'] = '0'
 
-    preload_cmd = remote_env.to_export_str()
-
     # Set the distro version for the utils/scripts/install-*.sh file
     distro_info = detect()
     distro = f"el{distro_info.version}"
@@ -129,41 +126,29 @@ def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False):
     elif "ubuntu" in distro_info.name.lower():
         distro = "ubuntu"
 
-    cmds = [f'{sys.executable} -m venv {mount_dir}/venv',
-            f'git clone https://github.com/daos-stack/daos.git {build_dir}',
-            f'git -C {build_dir} checkout {__get_daos_build_checkout(self)}',
-            f'git -C {build_dir} submodule update --init --recursive',
-            f'cp {build_dir}/utils/scripts/install-{distro}.sh /tmp/install.sh',
-            'sudo -E NO_OPENMPI_DEVEL=1 /tmp/install.sh -y',
-            'python3 -m pip install pip --upgrade',
-            f'python3 -m pip install -r {build_dir}/requirements-build.txt',
-            f'scons -C {build_dir} --jobs {build_jobs} --build-deps=only',
-            f'daos filesystem query {mount_dir}',
-            f'daos filesystem evict {build_dir}',
-            f'daos filesystem query {mount_dir}',
-            f'scons -C {build_dir} --jobs {build_jobs}',
-            f'scons -C {build_dir} --jobs {build_jobs} install --implicit-deps-unchanged',
-            f'daos filesystem query {mount_dir}']
-    for cmd in cmds:
-        command = '{} {}'.format(preload_cmd, cmd)
-        # Use a short timeout for most commands, but vary the build timeout based on dfuse mode.
-        timeout = 10 * 60
-        if cmd.startswith('scons'):
-            timeout = build_time * 60
-        elif cmd.endswith('install.sh -y'):
-            timeout *= 2
-        self.log_step(f"Running '{cmd}' with a {timeout}s timeout")
-        start = time.time()
-        result = run_remote(
-            self.log, self.hostlist_clients, command, verbose=True, timeout=timeout)
-        elapsed = time.time() - start
-        (minutes, seconds) = divmod(elapsed, 60)
-        self.log.info(
-            'Command %s completed in %d:%02d (%d%% of timeout)',
-            command, minutes, seconds, elapsed / timeout * 100)
-        if result.passed:
-            continue
-
+    command = " ".join([
+        remote_env.to_export_str(),
+        os.path.abspath(os.path.join(os.getcwd(), 'daos_build.sh')),
+        f"--python-cmd python{sys.version_info.major}.{sys.version_info.minor}",
+        f"--venv-dir {mount_dir}/venv",
+        f"--build-dir {build_dir}",
+        f"--mount-dir {mount_dir}",
+        f"--git-checkout {__get_daos_build_checkout(self)}",
+        f"--distro {distro}",
+        f"--build-jobs {build_jobs}",
+        "--debug"
+    ])
+    timeout = 14400  # 4 hours
+    self.log_step(f"Running '{command}' with a {timeout}s timeout")
+    start = time.time()
+    result = run_remote(
+        self.log, self.hostlist_clients, command, verbose=True, timeout=timeout)
+    elapsed = time.time() - start
+    (minutes, seconds) = divmod(elapsed, 60)
+    self.log.info(
+        'Command %s completed in %d:%02d (%d%% of timeout)',
+        command, minutes, seconds, elapsed / timeout * 100)
+    if not result.passed:
         self.log.info('Failure detected - debug information:')
         fail_type = 'Failure to build'
         if result.timeout:
@@ -172,9 +157,9 @@ def run_build_test(self, cache_mode, il_lib=None, run_on_vms=False):
             fail_type = 'Timeout building'
 
         self.log.error('BuildDaos Test Failed')
-        if cmd.startswith('scons'):
+        if 'scons' in result.all_stdout:
             run_remote(
-                self.log, self.hostlist_clients, 'cat {}/config.log'.format(build_dir), timeout=30)
+                self.log, self.hostlist_clients, f'cat {build_dir}/config.log', timeout=30)
         if il_lib is not None:
             self.fail(f'{fail_type} over dfuse with il in mode {cache_mode}')
         else: