Add ASAN mechanism for hip-tests (#3215)

Author: cjatin

build_tools/github_actions/test_executable_scripts/test_hiptests.py

@@ -3,31 +3,38 @@
 import shlex
 import subprocess
 from pathlib import Path
-import glob
+import platform
 import shutil
 import json
 import sys
 import platform
 
 logging.basicConfig(level=logging.INFO)
 THEROCK_BIN_DIR_STR = os.getenv("THEROCK_BIN_DIR")
-if THEROCK_BIN_DIR_STR is None:
-    logging.info(
-        "++ Error: env(THEROCK_BIN_DIR) is not set. Please set it before executing tests."
-    )
-    sys.exit(1)
 THEROCK_BIN_DIR = Path(THEROCK_BIN_DIR_STR)
 SCRIPT_DIR = Path(__file__).resolve().parent
 THEROCK_DIR = SCRIPT_DIR.parent.parent.parent
-SHARD_INDEX = os.getenv("SHARD_INDEX", 1)
-TOTAL_SHARDS = os.getenv("TOTAL_SHARDS", 1)
+SHARD_INDEX = int(os.getenv("SHARD_INDEX", 1)) - 1
+TOTAL_SHARDS = int(os.getenv("TOTAL_SHARDS", 1))
 AMDGPU_FAMILIES = os.getenv("AMDGPU_FAMILIES")
 os_type = platform.system().lower()
 CATCH_TESTS_PATH = str(Path(THEROCK_BIN_DIR).parent / "share" / "hip" / "catch_tests")
+
+# Importing is_asan from github_actions_utils.py
+sys.path.append(str(THEROCK_DIR / "build_tools" / "github_actions"))
+from github_actions_utils import is_asan
+
+env = os.environ.copy()
+
+if THEROCK_BIN_DIR_STR is None:
+    logging.info(
+        "++ Error: env(THEROCK_BIN_DIR) is not set. Please set it before executing tests."
+    )
+    sys.exit(1)
+
 if not os.path.isdir(CATCH_TESTS_PATH):
     logging.info(f"++ Error: catch tests not found in {CATCH_TESTS_PATH}")
     sys.exit(1)
-env = os.environ.copy()
 
 # TODO(#3204): Re-enable tests once issues are resolved
 TEST_TO_IGNORE = {
@@ -45,30 +52,18 @@
 }
 
 
-def get_test_count():
-    cmd = ["ctest", "--show-only=json-v1"]
+def get_asan_lib_path():
+    arch = platform.machine()
+    CLANG_PATH = str(Path(THEROCK_BIN_DIR).parent / "lib" / "llvm" / "bin" / "clang++")
+    cmd = [f"{CLANG_PATH}", f"--print-file-name=libclang_rt.asan-{arch}.so"]
+    logging.info(f"++ Exec [{CLANG_PATH}]$ {shlex.join(cmd)}")
     result = subprocess.run(
         cmd,
-        cwd=CATCH_TESTS_PATH,
         check=True,
+        text=True,
         capture_output=True,
     )
-    jdata = json.loads(result.stdout)
-    tests = jdata["tests"]
-    return len(tests)
-
-
-def get_test_range_per_shard(total_test_count: int, total_shards, shard_index):
-    tests_per_shard = int(total_test_count / total_shards)
-    current_index = (tests_per_shard * (shard_index - 1)) + 1
-    end_index = current_index + tests_per_shard - 1
-    if shard_index == total_shards:
-        # Retrieve remaining tests
-        end_index = total_test_count
-    logging.info(
-        f"""++ hip-tests ctest: shard {shard_index} / {total_shards}. Running:{tests_per_shard} tests"""
-    )
-    return [current_index, end_index]
+    return result.stdout.strip()
 
 
 def copy_dlls_exe_path():
@@ -103,24 +98,27 @@ def setup_env(env):
             env["LD_LIBRARY_PATH"] = f"{HIP_LIB_PATH}:{env['LD_LIBRARY_PATH']}"
         else:
             env["LD_LIBRARY_PATH"] = HIP_LIB_PATH
+        # For ASAN mode, we preload it for test count query and test running
+        if is_asan():
+            env["LD_PRELOAD"] = get_asan_lib_path()
+            # TODO: enable this when we have symbolizer patch in
+            # env["ASAN_SYMBOLIZER_PATH"] = str(Path(THEROCK_BIN_DIR).parent / "lib" / "llvm" / "bin" / "llvm-symbolizer")
     else:
         copy_dlls_exe_path()
 
 
 def execute_tests(env):
-    total_tests = get_test_count()
-    index_start, index_end = get_test_range_per_shard(
-        total_tests, int(TOTAL_SHARDS), int(SHARD_INDEX)
-    )
+    # Allow for more time in ASAN mode to run the tests.
+    timeout = 1500 if is_asan() else 600
     cmd = [
         "ctest",
-        "-I",
-        f"{index_start},{index_end}",
+        "--tests-information",
+        f"{SHARD_INDEX},,{TOTAL_SHARDS}",
         "--test-dir",
         CATCH_TESTS_PATH,
         "--output-on-failure",
         "--timeout",
-        "600",
+        f"{timeout}",
     ]
 
     if AMDGPU_FAMILIES in TEST_TO_IGNORE and os_type in TEST_TO_IGNORE[AMDGPU_FAMILIES]: