Upgrade custom libc++ to LLVM 19 and add sanitizer support to build_and_test.sh (#3131)

### Description of changes:
Currently, the custom libc++ used for MSAN/TSAN-instrumented builds is
pinned to LLVM 15, and running sanitizer builds requires manually
assembling the correct CMake flags. Additionally, the no-ASM code paths
are not tested under sanitizers in CI.

This change upgrades the custom libc++ from LLVM 15 to LLVM 19.1.7, adds
first-class `--sanitizer` support to `util/build_and_test.sh`, and
introduces new CI jobs that test the no-ASM code paths under ASan and
MSan.

**LLVM 19 upgrade:**
- Update `Dockerfile.al2023` to clone LLVM 19.1.7 (from 15.0.6)
- Bump `CXX_STANDARD` from 20 to 23 for the custom libc++ build (LLVM 19
added `expected.cpp` which requires C++23)
- Expand `util/bot/libcxx-config/__config_site` with required LLVM 19+
configuration macros (`_LIBCPP_HARDENING_MODE`, positive-sense feature
flags, thread API selection, ASan instrumentation flag)
- Add `util/bot/libcxx-config/__assertion_handler` (required by LLVM 19+
which no longer ships a default)
- Update `linux-multi-arch-omnibus.yml` to use the unversioned
`setup-clang.sh`

**Sanitizer support in `build_and_test.sh`:**
- Add `--sanitizer <name>` flag supporting `asan`, `msan`, `tsan`,
`ubsan`, and `cfi`
- Add `--test <binary>` flag to build and run a single test binary
directly
- Auto-clone LLVM (sparse, cached) when MSAN/TSAN need a custom libc++
and `LLVM_PROJECT_HOME` is not set
- Clean the build directory on sanitizer runs to avoid stale
configuration artifacts

**Bug fix:**
- Zero-initialize `res` and `tmp` arrays in `ec_nistp_scalar_mul` to
eliminate MSAN false positives on unused limbs

### Call-outs:
The new `sanitizer-tests` job in `actions-ci.yml` runs ASan and MSan
with `-DOPENSSL_NO_ASM=1` on x86-64 and aarch64. These jobs do **not**
overlap with the existing `sanitizers` job in
`linux-multi-arch-omnibus.yml`, which runs all five sanitizers (ASan,
MSan, TSan, UBSan, CFI) with ASM enabled inside Docker containers on
CodeBuild via `run_posix_sanitizers.sh`. The new jobs specifically cover
the no-ASM code paths under sanitizers, which were previously untested
in CI.

### Testing:
- New `sanitizer-tests` CI matrix covers ASan + MSan (no-ASM) × {x86-64,
aarch64}
- Existing Docker-based omnibus sanitizer jobs updated to use LLVM 19

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.

Author: justsmth

.github/docker_images/aws-lc/amazonlinux/Dockerfile.al2023

@@ -88,7 +88,7 @@ RUN <<EOF
 set -ex
 mkdir -p ${DEPENDENCIES_DIR}
 cd ${DEPENDENCIES_DIR}
-git clone https://github.com/llvm/llvm-project.git --branch llvmorg-15.0.6 --depth 1
+git clone https://github.com/llvm/llvm-project.git --branch llvmorg-19.1.7 --depth 1
 cd llvm-project
 rm -rf $(ls -A | grep -Ev "(libcxx|libcxxabi)")
 EOF

.github/workflows/actions-ci.yml

@@ -205,6 +205,74 @@ jobs:
       - name: Run tests
         run: cmake --build ./build --target run_tests
 
+  sanitizer-tests:
+    name: sanitizer - ${{ matrix.arch.name }} - ${{ matrix.config.name }} [${{ matrix.shard_index }}/${{ matrix.config.total_shards }}]
+    needs: [sanity-test-run]
+    if: github.repository_owner == 'aws'
+    runs-on: ${{ matrix.arch.runner }}
+    env:
+      GOFLAGS: "-buildvcs=false"
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - name: x86-64
+            runner: ubuntu-latest
+          - name: aarch64
+            runner: ubuntu-24.04-arm
+        config:
+          - name: asan-noasm
+            sanitizer: asan
+            extra_cmake_args: "-DOPENSSL_NO_ASM=1"
+            total_shards: 1
+          - name: msan-noasm
+            sanitizer: msan
+            extra_cmake_args: "-DOPENSSL_NO_ASM=1"
+            total_shards: 4
+        shard_index: [0, 1, 2, 3]
+        exclude:
+          - config:
+              name: asan-noasm
+        include:
+          - arch:
+              name: x86-64
+              runner: ubuntu-latest
+            config:
+              name: asan-noasm
+              sanitizer: asan
+              extra_cmake_args: "-DOPENSSL_NO_ASM=1"
+              total_shards: 1
+            shard_index: 0
+          - arch:
+              name: aarch64
+              runner: ubuntu-24.04-arm
+            config:
+              name: asan-noasm
+              sanitizer: asan
+              extra_cmake_args: "-DOPENSSL_NO_ASM=1"
+              total_shards: 1
+            shard_index: 0
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ">=1.18"
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ninja-build
+      - name: Build and Test crypto_test
+        env:
+          GTEST_TOTAL_SHARDS: ${{ matrix.config.total_shards }}
+          GTEST_SHARD_INDEX: ${{ matrix.shard_index }}
+        run: |
+          ./util/build_and_test.sh --sanitizer ${{ matrix.config.sanitizer }} \
+            --test crypto_test --build-target all_tests \
+            -DCMAKE_BUILD_TYPE=Release ${{ matrix.config.extra_cmake_args }}
+      - name: Run ssl_test
+        if: matrix.shard_index == 0
+        run: ./build/ssl/ssl_test
+
   musl-tests:
     name: musl - ${{ matrix.config.arch }}
     needs: [sanity-test-run]

.github/workflows/linux-multi-arch-omnibus.yml

@@ -512,7 +512,7 @@ jobs:
           env: |
             AWS_LC_GO_TEST_TIMEOUT
           run: |
-            source /opt/compiler-env/setup-clang-15.sh
+            source /opt/compiler-env/setup-clang.sh
             ./tests/ci/run_posix_sanitizers.sh
 
   benchmark:

CMakeLists.txt

@@ -1174,8 +1174,9 @@ if(USE_CUSTOM_LIBCXX)
   set_target_properties(
     libcxx libcxxabi PROPERTIES
     COMPILE_FLAGS "-Wno-missing-prototypes -Wno-implicit-fallthrough"
-    # libc++ and libc++abi must be built in C++20 mode.
-    CXX_STANDARD 20
+    # libc++ and libc++abi must be built in C++23 mode.
+    # LLVM 19+ added expected.cpp which requires C++23.
+    CXX_STANDARD 23
     CXX_STANDARD_REQUIRED TRUE
   )
   # libc++abi depends on libc++ internal headers.

crypto/fips_callback_test.cc

@@ -106,8 +106,8 @@ TEST(FIPSCallback, PowerOnSelfTests) {
   uint8_t private_key[ED25519_PRIVATE_KEY_LEN];
   ED25519_keypair(public_key, private_key);
 
-  uint8_t message[2];
-  uint8_t context[2];
+  uint8_t message[2] = {0};
+  uint8_t context[2] = {0};
   uint8_t signature[ED25519_SIGNATURE_LEN];
   EXPECT_TRUE(ED25519ph_sign(signature, message, sizeof(message), private_key, context, sizeof(context)));
 

crypto/fipsmodule/ec/ec_nistp.c

@@ -440,8 +440,8 @@ void ec_nistp_scalar_mul(const ec_nistp_meth *ctx,
   // to avoid allocation, and just take pointers to individual coordinates.
   // (This cruft will dissapear when we refactor point_add/dbl to work with
   // whole points instead of individual coordinates).
-  ec_nistp_felem_limb res[3 * FELEM_MAX_NUM_OF_LIMBS];
-  ec_nistp_felem_limb tmp[3 * FELEM_MAX_NUM_OF_LIMBS];
+  ec_nistp_felem_limb res[3 * FELEM_MAX_NUM_OF_LIMBS] = {0};
+  ec_nistp_felem_limb tmp[3 * FELEM_MAX_NUM_OF_LIMBS] = {0};
   ec_nistp_felem_limb *x_res = &res[0];
   ec_nistp_felem_limb *y_res = &res[ctx->felem_num_limbs];
   ec_nistp_felem_limb *z_res = &res[ctx->felem_num_limbs * 2];

util/bot/libcxx-config/__assertion_handler

@@ -0,0 +1,10 @@
+#ifndef BORINGSSL_LIBCXX_ASSERTION_HANDLER_
+#define BORINGSSL_LIBCXX_ASSERTION_HANDLER_
+
+#include <__verbose_abort>
+
+// We only use our custom libc++ for sanitizer testing, so we can use verbose
+// aborts for assertion failures.
+#define _LIBCPP_ASSERTION_HANDLER(message) _LIBCPP_VERBOSE_ABORT("%s", message)
+
+#endif  // BORINGSSL_LIBCXX_ASSERTION_HANDLER_

util/bot/libcxx-config/__config_site

@@ -1,6 +1,47 @@
 #ifndef BORINGSSL_LIBCXX_CONFIG_SITE_
 #define BORINGSSL_LIBCXX_CONFIG_SITE_
 
+// We only use our custom libc++ for sanitizer testing, so enable all hardening
+// checks. This is required for LLVM 16+ which mandates that
+// _LIBCPP_HARDENING_MODE is set to a valid value.
+#define _LIBCPP_HARDENING_MODE _LIBCPP_HARDENING_MODE_DEBUG
+
+// libc++ headers disable std::string ASan annotations if this is not defined.
+// This is to avoid false positives when libc++'s runtime components are
+// uninstrumented. When building our custom libc++, libc++ will be as
+// instrumented as the caller, so we can safely enable this.
+#define _LIBCPP_INSTRUMENTED_WITH_ASAN 1
+
+// New-style feature macros (LLVM 19+). Older versions use
+// _LIBCPP_HAS_NO_<feature> negated macros which are auto-detected, so these
+// definitions are only consumed by the versions that require them.
+#define _LIBCPP_HAS_FILESYSTEM 1
+#define _LIBCPP_HAS_LOCALIZATION 1
+#define _LIBCPP_HAS_MONOTONIC_CLOCK 1
+#define _LIBCPP_HAS_RANDOM_DEVICE 1
+#define _LIBCPP_HAS_THREADS 1
+#define _LIBCPP_HAS_UNICODE 1
+#define _LIBCPP_HAS_WIDE_CHARACTERS 1
+
+// LLVM 19+ replaced _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS with a
+// positive-sense macro.
+#define _LIBCPP_HAS_VENDOR_AVAILABILITY_ANNOTATIONS 0
+// Keep the old macro for LLVM <= 18 compatibility.
 #define _LIBCPP_HAS_NO_VENDOR_AVAILABILITY_ANNOTATIONS
 
-#endif  // BORINGSSL_LIBCXX_CONFIG_SITE_
+#if defined(__APPLE__)
+#define _LIBCPP_PSTL_BACKEND_LIBDISPATCH 1
+#else
+#define _LIBCPP_PSTL_BACKEND_STD_THREAD 1
+#endif
+
+// Thread API macros use defined() checks in LLVM 19+, so only define the one
+// that applies. Defining the others to 0 would still satisfy defined() and
+// break the include logic.
+#ifdef _WIN32
+#define _LIBCPP_HAS_THREAD_API_WIN32 1
+#else
+#define _LIBCPP_HAS_THREAD_API_PTHREAD 1
+#endif
+
+#endif  // BORINGSSL_LIBCXX_CONFIG_SITE_

util/build_and_test.sh

@@ -13,8 +13,38 @@ DEFAULT_CMAKE_FLAGS=("-DCMAKE_BUILD_TYPE=Debug" "-GNinja")
 BUILD_TARGET="all_tests"
 RUN_TARGET="run_tests"
 BUILD_ONLY=false
+SANITIZER=""
+TEST_BINARY=""
 CMAKE_ARGS=()
 
+# Ensures LLVM_PROJECT_HOME is set, cloning if necessary.
+# MSAN and TSAN require an MSAN-instrumented libc++ (USE_CUSTOM_LIBCXX), which
+# is built from LLVM source. If LLVM_PROJECT_HOME is not already set, this
+# function performs a minimal sparse clone of the LLVM project (libcxx and
+# libcxxabi only) and caches it under /tmp for subsequent runs.
+setup_llvm_project() {
+  if [[ -n "${LLVM_PROJECT_HOME:-}" ]]; then
+    echo "Using LLVM_PROJECT_HOME=${LLVM_PROJECT_HOME}"
+    return
+  fi
+
+  local llvm_cache_dir="/tmp/aws-lc-llvm-project"
+  if [[ -d "${llvm_cache_dir}/libcxx" && -d "${llvm_cache_dir}/libcxxabi" ]]; then
+    echo "Using cached LLVM project at ${llvm_cache_dir}"
+  else
+    echo "LLVM_PROJECT_HOME not set. Cloning llvm-project to ${llvm_cache_dir}..."
+    rm -rf "${llvm_cache_dir}"
+    git clone https://github.com/llvm/llvm-project.git \
+      --branch llvmorg-19.1.7 --depth 1 \
+      --filter=blob:none --sparse \
+      "${llvm_cache_dir}"
+    pushd "${llvm_cache_dir}"
+    git sparse-checkout set libcxx libcxxabi
+    popd
+  fi
+  export LLVM_PROJECT_HOME="${llvm_cache_dir}"
+}
+
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --build-only)
@@ -29,6 +59,15 @@ while [[ $# -gt 0 ]]; do
       RUN_TARGET="$2"
       shift 2
       ;;
+    --sanitizer)
+      SANITIZER="$2"
+      shift 2
+      ;;
+    --test)
+      TEST_BINARY="$2"
+      BUILD_TARGET="$2"
+      shift 2
+      ;;
     *)
       # Everything else gets passed to CMake
       CMAKE_ARGS+=("$1")
@@ -37,13 +76,60 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
+# Handle sanitizer configuration
+if [[ -n "${SANITIZER}" ]]; then
+  # Sanitizer builds require a clean build directory to avoid stale artifacts
+  # from a different configuration (e.g., switching from ASAN to MSAN).
+  rm -rf "${AWS_LC_BUILD}"
+  export AWS_LC_GO_TEST_TIMEOUT="${AWS_LC_GO_TEST_TIMEOUT:-90m}"
+
+  # All sanitizers require Clang.
+  export CC=${CC:-clang}
+  export CXX=${CXX:-clang++}
+
+  case "${SANITIZER}" in
+    asan)
+      CMAKE_ARGS+=("-DASAN=1")
+      ;;
+    msan)
+      CMAKE_ARGS+=("-DMSAN=1" "-DUSE_CUSTOM_LIBCXX=1")
+      setup_llvm_project
+      CMAKE_ARGS+=("-DLLVM_PROJECT_HOME=${LLVM_PROJECT_HOME}")
+      ;;
+    tsan)
+      CMAKE_ARGS+=("-DTSAN=1" "-DUSE_CUSTOM_LIBCXX=1")
+      setup_llvm_project
+      CMAKE_ARGS+=("-DLLVM_PROJECT_HOME=${LLVM_PROJECT_HOME}")
+      ;;
+    ubsan)
+      CMAKE_ARGS+=("-DUBSAN=1")
+      ;;
+    cfi)
+      CMAKE_ARGS+=("-DCFI=1")
+      ;;
+    *)
+      echo "Error: Unknown sanitizer '${SANITIZER}'. Supported: asan, msan, tsan, ubsan, cfi"
+      exit 1
+      ;;
+  esac
+fi
+
 mkdir -p "${AWS_LC_BUILD}"
 
 cmake "${BASE_DIR}" -B "${AWS_LC_BUILD}" "${DEFAULT_CMAKE_FLAGS[@]}" "${CMAKE_ARGS[@]}"
 
 cmake --build "${AWS_LC_BUILD}" -j --target ${BUILD_TARGET}
 
-# Only run tests if --build-only was not specified
-if [ "$BUILD_ONLY" = false ]; then
+if [[ -n "${TEST_BINARY}" ]]; then
+  # --test was specified: find and run the test binary directly instead of
+  # going through the heavyweight run_tests CMake target.
+  TEST_PATH=$(find "${AWS_LC_BUILD}" -name "${TEST_BINARY}" -type f -executable | head -1)
+  if [[ -z "${TEST_PATH}" ]]; then
+    echo "Error: Could not find test binary '${TEST_BINARY}' in ${AWS_LC_BUILD}"
+    exit 1
+  fi
+  "${TEST_PATH}"
+elif [ "$BUILD_ONLY" = false ]; then
+  # Default: run via the CMake run_tests target (Go tests + SSL runner + unit tests).
   cmake --build "${AWS_LC_BUILD}" --target ${RUN_TARGET}
 fi