Merge pull request #25120 from rockwotj/clang-fixes-bazel

Author: rockwotj

.bazelrc

@@ -14,8 +14,6 @@ build --@rules_go//go/config:pure
 # https://github.com/bazel-contrib/toolchains_llvm/pull/445#issuecomment-2605443516
 build --copt=-ffile-compilation-dir=. --host_copt=-ffile-compilation-dir=.
 
-common:clang-19 --extra_toolchains=@llvm_19_toolchain//:all
-
 build:system-clang --extra_toolchains=@local_config_cc_toolchains//:all
 build:system-clang --action_env=BAZEL_COMPILER=clang
 build:system-clang --cxxopt=-std=c++23 --host_cxxopt=-std=c++23

MODULE.bazel

@@ -96,6 +96,12 @@ use_repo(non_module_dependencies, "x86_64_sysroot")
 use_repo(non_module_dependencies, "aarch64_sysroot")
 
 bazel_dep(name = "toolchains_llvm", version = "1.3.0")
+archive_override(
+    module_name = "toolchains_llvm",
+    integrity = "sha256-tiSkUD4G/Zxkdm+TPSclElauijatg7jF9eqs8LvD6hw=",
+    strip_prefix = "toolchains_llvm-e831f94a8b7f3a39391f5822adcab8e4d443c03b",
+    urls = "https://github.com/bazel-contrib/toolchains_llvm/archive/e831f94a8b7f3a39391f5822adcab8e4d443c03b.tar.gz",
+)
 
 # ====================================
 # LLVM toolchain
@@ -129,46 +135,18 @@ COMPILE_FLAGS = {
     ] + COMMON_COMPILE_FLAGS,
 }
 
-llvm.toolchain(
-    name = "llvm_19_toolchain",
-    compile_flags = COMPILE_FLAGS,
-    cxx_standard = {"": CXX_STANDARD},
-    llvm_version = "19.1.1",
-    sha256 = {
-        "linux-aarch64": "c5cade3af930255296e097687ed9c86c660ca5b619a352a536353aaa5d4fde2e",
-        "linux-x86_64": "0617e50fdb484bcd2169b24acd90e9b9fec7e6730d187fe8d2acfe69d2d429c4",
-    },
-    urls = {
-        "linux-aarch64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/llvm-19.1.1-ubuntu-22.04-aarch64.tar.gz"],
-        "linux-x86_64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/llvm-19.1.1-ubuntu-22.04-x86_64.tar.gz"],
-    },
-)
-llvm.sysroot(
-    name = "llvm_19_toolchain",
-    label = "@x86_64_sysroot//:sysroot",
-    targets = ["linux-x86_64"],
-)
-llvm.sysroot(
-    name = "llvm_19_toolchain",
-    label = "@aarch64_sysroot//:sysroot",
-    targets = ["linux-aarch64"],
-)
-use_repo(llvm, "llvm_19_toolchain", "llvm_19_toolchain_llvm")
-
-register_toolchains("@llvm_19_toolchain//:all")
-
 llvm.toolchain(
     name = "llvm_18_toolchain",
     compile_flags = COMPILE_FLAGS,
     cxx_standard = {"": CXX_STANDARD},
     llvm_version = "18.1.8",
     sha256 = {
-        "linux-aarch64": "95223a9fd35a2cfea2582b34519bbf65ff6bed498a378eb545a9d4fb8ee5a832",
-        "linux-x86_64": "ca33f2b189bf56a3266101b6ed267739cf10d311b5c3fbdd2c05a306574f6f70",
+        "linux-aarch64": "621856db24f9b081d3d097f7b51b33f8cf096b0d103214b23c49fd5978ce2354",
+        "linux-x86_64": "b568ffa3ae21ac57b4dc5c96743b298343cc503cfb52192155f836b3b10213cf",
     },
     urls = {
-        "linux-aarch64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/llvm-18.1.8-ubuntu-22.04-aarch64.tar.gz"],
-        "linux-x86_64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/pgo/llvm-18.1.8-ubuntu-22.04-x86_64.tar.gz"],
+        "linux-aarch64": ["https://github.com/redpanda-data/llvm-project/releases/download/llvmorg-18.1.8/llvm-18.1.8-ubuntu-22.04-aarch64-2025-02-25.tar.zst"],
+        "linux-x86_64": ["https://github.com/redpanda-data/llvm-project/releases/download/llvmorg-18.1.8/llvm-18.1.8-ubuntu-22.04-x86_64-2025-02-25.tar.zst"],
     },
 )
 llvm.sysroot(

MODULE.bazel.lock

@@ -199,13 +199,13 @@ filegroup(
     http_archive(
         name = "x86_64_sysroot",
         build_file_content = sysroot_build_file,
-        sha256 = "7639199d529a2aee4b0007a2c55f6178235e732782ca47795a3022342e7ad985",
-        urls = ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/sysroot-ubuntu-22.04-x86_64-2025-01-26.tar.gz"],
+        sha256 = "282b7eb89ca45d2309217d5d2099cc087c1e7bd55f7891b9d2ddca648b6663b7",
+        urls = ["https://github.com/redpanda-data/llvm-project/releases/download/llvmorg-19.1.7/sysroot-ubuntu-22.04-x86_64-2025-02-24.tar.zst"],
     )
 
     http_archive(
         name = "aarch64_sysroot",
         build_file_content = sysroot_build_file,
-        sha256 = "536cc7a4e0eb41418239da62f7f8d7205f1006fa5670f4ab8fab4a8db86de087",
-        urls = ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/sysroot-ubuntu-22.04-aarch64-2025-01-26.tar.gz"],
+        sha256 = "a162b62d4d34588232137c8f14d45849aaa36202b9081f2718452f4a086ae817",
+        urls = ["https://github.com/redpanda-data/llvm-project/releases/download/llvmorg-19.1.7/sysroot-ubuntu-22.04-aarch64-2025-02-24.tar.zst"],
     )

bazel/repositories.bzl

@@ -426,17 +426,18 @@ def redpanda_cc_bench(
             "exec $@ --iterations=1 --runs=1 --duration=0 --no-stdout --overprovisioned",
         ],
     )
+    test_data, test_env = _test_options()
     native.sh_test(
         name = name + "_test",
         timeout = timeout,
         tags = resource_tags + tags,
         srcs = [name + "_test_script"],
-        env = env,
+        env = env | test_env,
         args = [
             "$(rootpath :{})".format(name),
         ] + args,
         data = [
             ":" + name,
-        ] + data,
+        ] + data + test_data,
         target_compatible_with = target_compatible_with,
     )

bazel/test.bzl

@@ -1 +1,2 @@
 *.tar.gz
+*.tar.zst

bazel/toolchain/.gitignore

@@ -14,7 +14,7 @@ ADD --chmod=755 https://github.com/kadwanev/retry/raw/20997c7712a4/retry /usr/bi
 
 WORKDIR /opt
 
-ARG LLVM_VERSION=19.1.6
+ARG LLVM_VERSION=19.1.7
 
 RUN git clone --branch llvmorg-${LLVM_VERSION} --single-branch https://github.com/llvm/llvm-project.git
 
@@ -41,10 +41,13 @@ cmake -G Ninja -DLLVM_TARGETS_TO_BUILD='AArch64;X86' \\
   -DLLVM_ENABLE_PIC=ON \\
   -DLLVM_INCLUDE_TESTS=OFF \\
   -DLLVM_INCLUDE_EXAMPLES=OFF \\
+  -DLLVM_INCLUDE_BENCHMARKS=OFF \\
+  -DLLVM_ENABLE_BINDINGS=OFF \\
   -DLLVM_INCLUDE_UTILS=OFF \\
   -DLLVM_INCLUDE_DOCS=OFF \\
-  -DLLVM_ENABLE_ZLIB=OFF \\
-  -DLLVM_ENABLE_ZSTD=OFF \\
+  -DLLVM_ENABLE_ZLIB=ON \\
+  -DLLVM_ENABLE_ZSTD=ON \\
+  -DLLVM_USE_STATIC_ZSTD=TRUE \\
   -DLLVM_ENABLE_Z3_SOLVER=OFF \\
   -DLLVM_ENABLE_DIA_SDK=OFF \\
   -DLLVM_ENABLE_LIBEDIT=OFF \\
@@ -74,10 +77,11 @@ cmake -G Ninja -DLLVM_TARGETS_TO_BUILD='AArch64;X86' \\
   -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \\
   -DLLVM_PARALLEL_COMPILE_JOBS=${MAX_COMPILER_JOBS} \\
   -DLLVM_PARALLEL_LINK_JOBS=${MAX_LINKER_JOBS} \\
-  -DLIBCXX_ABI_VERSION=2 \\
+  -DLIBCXX_ABI_UNSTABLE=ON \\
   -DLLVM_ENABLE_LIBCXX=ON \\
   -DLLVM_STATIC_LINK_CXX_STDLIB=ON \\
   -DLLVM_ENABLE_LTO=On \\
+  -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON \\
   "\$@"
 
 retry --sleep=0 --tries=100 -- ninja -C "\$BUILD_DIR"
@@ -96,11 +100,9 @@ RUN /opt/build.sh \
 
 WORKDIR /opt/install
 
-RUN bash -c 'tar -czvf /opt/llvm-${LLVM_VERSION}-$(source /etc/os-release && echo -n "$ID")-$(source /etc/os-release && echo -n "$VERSION_ID")-$(uname -m).tar.gz .'
-
 FROM scratch AS lto
 
-COPY --from=lto-build /opt/llvm-*.tar.gz .
+COPY --from=lto-build /opt/install/ .
 
 FROM base AS pgo-build
 
@@ -113,8 +115,9 @@ RUN /opt/build.sh \
   -DLLVM_BUILD_INSTRUMENTED=IR \
   -DLLVM_PROFILE_DATA_DIR=/tmp/clang-pgo-data/ \
   -DLLVM_ENABLE_PROJECTS="clang;lld;bolt" \
-  -DCMAKE_C_FLAGS="-Xclang -mllvm -Xclang -vp-counters-per-site=6" \
-  -DCMAKE_CXX_FLAGS="-Xclang -mllvm -Xclang -vp-counters-per-site=6" \
+  -DLLVM_VP_COUNTERS_PER_SITE=6 \
+  -DLLVM_BUILD_LLVM_DYLIB=ON \
+  -DLLVM_LINK_LLVM_DYLIB=ON \
   -DLLVM_BUILD_RUNTIME=No \
   -DLLVM_ENABLE_LTO=${LTO_MODE}
 
@@ -135,13 +138,24 @@ llvm.toolchain_root(
     name = "pgo_toolchain",
     path = "/opt/stage-1",
 )
+llvm.sysroot(
+    name = "pgo_toolchain",
+    label = "@x86_64_sysroot//:sysroot",
+    targets = ["linux-x86_64"],
+)
+llvm.sysroot(
+    name = "pgo_toolchain",
+    label = "@aarch64_sysroot//:sysroot",
+    targets = ["linux-aarch64"],
+)
 use_repo(llvm, "pgo_toolchain")
 EOF
 RUN cat pgo_compiler.bzl >> MODULE.bazel
 
-RUN retry --sleep=0 --tries=100 -- bazel build \
+RUN bazel build \
   --config=release \
   --extra_toolchains=@pgo_toolchain//:all \
+  --nokeep_going \
   --copt=-g \
   --copt=-w \
   --copt=-Wno-error \
@@ -158,12 +172,12 @@ RUN llvm-profdata-${SYSTEM_CLANG_VERSION} merge -output=/tmp/profdata.prof /tmp/
 RUN /opt/build.sh \
   -DCMAKE_INSTALL_PREFIX=/opt/stage-2 \
   -DLLVM_ENABLE_LTO=${LTO_MODE} \
+  -DLLVM_BUILD_LLVM_DYLIB=ON \
+  -DLLVM_LINK_LLVM_DYLIB=ON \
   -DLLVM_PROFDATA_FILE=/tmp/profdata.prof
 
-WORKDIR /opt/stage-2
-
-RUN bash -c 'tar -czvf /opt/llvm-${LLVM_VERSION}-$(source /etc/os-release && echo -n "$ID")-$(source /etc/os-release && echo -n "$VERSION_ID")-$(uname -m).tar.gz .'
+# TODO: only copy binaries from stage 2
 
 FROM scratch AS pgo
 
-COPY --from=pgo-build /opt/llvm-*.tar.gz .
+COPY --from=pgo-build /opt/stage-2/ .