DAOS-19102 test: Stabilize NLT memcheck testing of Go binaries

Jenkinsfile

@@ -579,6 +579,20 @@ pipeline {
                                                       ' PREFIX=/opt/daos TARGET_TYPE=release'))
                             sh label: 'Generate RPMs',
                                 script: './ci/rpm/gen_rpms.sh el9 "' + env.DAOS_RELVAL + '"'
+                            // For non-release builds, create a separate build with the valgrind
+                            // tag for NLT memcheck testing.  This is necessary to avoid problems
+                            // caused by valgrind being confused by the Go runtime. We don't want
+                            // to use the valgrind build for normal testing because it is much slower.
+                            // BUILD_TYPE=dev is set for PR/dev builds in sconsArgs(), and
+                            // TARGET_TYPE=release is used to select pre-built cached prerequisites.
+                            job_step_update(
+                                sconsBuild(parallel_build: true,
+                                           build_deps: 'no',
+                                           scons_args: sconsArgs() +
+                                                      ' BUILD_VALGRIND=1 PREFIX=/opt/daos TARGET_TYPE=release'))
+                            sh label: 'Stash valgrind install tree for NLT',
+                                script: 'tar -C / -cf opt-daos-valgrind.tar opt/daos'
+                            stash(name: 'opt-daos-valgrind', includes: 'opt-daos-valgrind.tar')
                         }
                     }
                     post {
@@ -715,16 +729,22 @@ pipeline {
                         label params.CI_NLT_1_LABEL
                     }
                     steps {
+                        // NLT memchecks the valgrind-tagged build, not the shared -race one.
+                        unstash 'opt-daos-valgrind'
                         job_step_update(
-                            unitTest(timeout_time: 60,
+                            unitTest(timeout_time: 60 * cachedCommitPragma(pragma: 'NLT-repeat', def_val: '1').toInteger(),
                                      inst_repos: daosRepos(),
                                      test_script: 'ci/unit/test_nlt.sh' +
                                                   ' --system-ram-reserved 4' +
                                                   ' --max-log-size 1950MiB' +
                                                   ' --dfuse-dir /localhome/jenkins/' +
                                                   ' --log-usage-save nltir.xml' +
                                                   ' --log-usage-export nltr.json' +
-                                                  ' --class-name nlt all',
+                                                  ' --class-name nlt' +
+                                                  " --repeat ${cachedCommitPragma(pragma: 'NLT-repeat', def_val: '1')}" +
+                                                  /* groovylint-disable-next-line LineLength */
+                                                  (cachedCommitPragma(pragma: 'NLT-repeat-failfast', def_val: 'false').toLowerCase() == 'true' ? ' --failfast' : '') +
+                                                  ' all',
                                      with_valgrind: 'memcheck',
                                      valgrind_pattern: '*memcheck.xml',
                                      always_script: 'ci/unit/test_nlt_post.sh',

ci/unit/test_nlt.sh

@@ -10,8 +10,14 @@ rm -rf dnt.*.memcheck.xml vm_test/
 NODE=${NODELIST%%,*}
 mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 
-# Copy over the install tree and some of the build tree.
-rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos.tar utils requirements-utest.txt jenkins@"$NODE":build/
+# Copy over the install tree and some of the build tree. The memcheck NLT stage
+# ships the valgrind-tagged build (opt-daos-valgrind.tar); the fault-injection
+# stage ships the standard opt-daos.tar. Use whichever was unstashed.
+opt_tar=opt-daos.tar
+[ -f opt-daos-valgrind.tar ] && opt_tar=opt-daos-valgrind.tar
+rm -f opt-daos-install.tar
+ln "$opt_tar" opt-daos-install.tar
+rsync -rlpt -z -e "ssh $SSH_KEY_ARGS" .build_vars* opt-daos-install.tar utils requirements-utest.txt jenkins@"$NODE":build/
 
 ssh -T "$SSH_KEY_ARGS" jenkins@"$NODE" \
     "DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \

ci/unit/test_nlt_node.sh

@@ -12,7 +12,9 @@ if [ "$(sudo sysctl -n vm.max_map_count)" -lt "1000000" ] ; then
 fi
 
 cd build
-tar -xf opt-daos.tar
+# test_nlt.sh ships whichever build (memcheck valgrind vs fault-injection)
+# under the canonical name opt-daos-install.tar.
+tar -xf opt-daos-install.tar
 sudo mv opt/daos /opt/
 
 # Setup daos admin etc.

site_scons/prereq_tools/base.py

@@ -516,6 +516,10 @@ def __init__(self, env, opts):
         opts.Add(EnumVariable('WARNING_LEVEL', "Set default warning level", 'error',
                               ['warning', 'warn', 'error'], ignorecase=2))
         opts.Add(('SANITIZERS', 'Instrument C code with Google Sanitizers', None))
+        opts.Add(BoolVariable('BUILD_VALGRIND',
+                              'Build Go artifacts with the Go "valgrind" tag for Memcheck '
+                              '(also drops -race; ignored for release)',
+                              False))
         opts.Add(BoolVariable('CMOCKA_FILTER_SUPPORTED', 'Allows to filter cmocka tests', False))
         opts.Add(BoolVariable('CRT_PP', 'Preprocess CaRT sources', False))
         opts.Add(BoolVariable('HEAP_PROFILER', 'Instrument C code with Gperftools Heap Profiler',

site_scons/site_tools/go_builder.py

@@ -11,6 +11,21 @@
 include_re = re.compile(r'\#include [<"](\S+[>"])', re.M)
 
 
+def _is_valgrind_build(env):
+    """Return True if Go artifacts should be built with the Go 1.25+ "valgrind" tag.
+
+    BUILD_VALGRIND=1 makes the Go runtime cooperate with Memcheck. Ignored for
+    release builds.
+    """
+    if not env.get('BUILD_VALGRIND'):
+        return False
+    if env.get('BUILD_TYPE') == 'release':
+        return False
+    if env.get('SANITIZERS'):
+        Exit('BUILD_VALGRIND=1 is incompatible with SANITIZERS')
+    return True
+
+
 def _scan_go_file(node, env, _path):
     """Scanner for go code"""
     src_dir = os.path.dirname(str(node))
@@ -119,6 +134,7 @@ def _check_go_version(context):
         return 1
 
     env.d_go_bin = env.get("GO_BIN", env.WhereIs(GO_COMPILER, os.environ['PATH']))
+    env.AddMethod(_is_valgrind_build, 'd_is_valgrind_build')
 
     if GetOption('help') or GetOption('clean'):
         return