Merge remote-tracking branch 'origin/release/26.04' into valid_if-to-bools_to_mask

Author: mythrocks

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "thirdparty/cudf"]
 	path = thirdparty/cudf
 	url = https://github.com/rapidsai/cudf.git
-	branch = main
+	branch = release/26.04

build/Dockerfile.devel

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###
+# Build the image for spark-rapids-jni local development environment.
+
+# Inherit from the CICD docker image
+ARG JNI_DOCKER_CICD_IMAGE
+FROM ${JNI_DOCKER_CICD_IMAGE}
+
+# Validate the CICD docker image is set
+ARG JNI_DOCKER_CICD_IMAGE
+RUN if [ -z "$JNI_DOCKER_CICD_IMAGE" ]; then \
+      echo "Error: JNI_DOCKER_CICD_IMAGE is not set"; exit 1; \
+    fi
+
+# Enable the gcc-toolset by default for bash shell
+RUN echo "source scl_source enable gcc-toolset-${TOOLSET_VERSION}" >> /etc/bashrc
+
+# Execute every time a new non-interactive bash shell is started
+ENV BASH_ENV=/etc/bashrc
+
+### Remove the old version of git
+RUN rpm -e --nodeps git-core git || true
+
+# Install a newer version of git to support worktree feature
+ARG GIT_VERSION=2.53.0
+ARG PARALLEL_LEVEL
+
+# Install the dependencies for building git
+RUN dnf install -y libcurl-devel openssl-devel pcre2-devel && dnf clean all
+
+RUN mkdir -p /tmp/git-src && cd /tmp/git-src && \
+    wget -O "git-${GIT_VERSION}.tar.xz" "https://www.kernel.org/pub/software/scm/git/git-${GIT_VERSION}.tar.xz" && \
+    tar -xf "git-${GIT_VERSION}.tar.xz" && \
+    cd "git-${GIT_VERSION}" && \
+    scl enable gcc-toolset-${TOOLSET_VERSION} -- bash -c " \
+      ./configure \
+        --prefix=/usr \
+        --with-libpcre2 \
+        --with-openssl \
+        --with-curl && \
+      make -j\"${PARALLEL_LEVEL:-\$(nproc)}\" all && \
+      make install" && \
+      cd / && rm -rf /tmp/git-src
+

build/run-in-docker

@@ -33,8 +33,10 @@ GIT_COMMON_DIR_REL=$(git rev-parse --git-common-dir)
 GIT_COMMON_DIR=$(realpath "$GIT_COMMON_DIR_REL")
 WORKDIR=${WORKDIR:-$REPODIR}
 TZ=${TZ:-UTC}
+PARALLEL_LEVEL=${PARALLEL_LEVEL:-$(nproc)}
 
 CUDA_VERSION=${CUDA_VERSION:-12.9.1}
+OS_RELEASE=${OS_RELEASE:-8}
 DOCKER_CMD=${DOCKER_CMD:-docker}
 DOCKER_BUILD_EXTRA_ARGS=${DOCKER_BUILD_EXTRA_ARGS:-""}
 if [ "$(uname -m)" == "aarch64" ]; then
@@ -46,23 +48,31 @@ DOCKER_RUN_EXTRA_ARGS=${DOCKER_RUN_EXTRA_ARGS:-""}
 LOCAL_CCACHE_DIR=${LOCAL_CCACHE_DIR:-"$HOME/.ccache"}
 LOCAL_MAVEN_REPO=${LOCAL_MAVEN_REPO:-"$HOME/.m2/repository"}
 
-if [ "$JNI_DOCKER_DEV_BUILD" == "ON" ]; then
-  echo "Building docker image for local development, gcc-toolset is enabled by default..."
-  JNI_DOCKER_IMAGE="spark-rapids-jni-build:${CUDA_VERSION}-devel-rockylinux8"
-else
-  echo "Building docker image for production, gcc-toolset is NOT enabled by default..."
-  JNI_DOCKER_IMAGE="spark-rapids-jni-build:${CUDA_VERSION}-rockylinux8"
-fi
-
 # ensure directories exist
 mkdir -p "$LOCAL_CCACHE_DIR" "$LOCAL_MAVEN_REPO"
 
+echo "Building docker image for CICD environment..."
+JNI_DOCKER_CICD_IMAGE="spark-rapids-jni-build:${CUDA_VERSION}-rockylinux${OS_RELEASE}"
 $DOCKER_CMD build $DOCKER_BUILD_EXTRA_ARGS -f $REPODIR/ci/Dockerfile \
   --build-arg CUDA_VERSION=$CUDA_VERSION \
-  --build-arg DEV_BUILD=$JNI_DOCKER_DEV_BUILD \
-  -t $JNI_DOCKER_IMAGE \
+  --build-arg OS_RELEASE=$OS_RELEASE \
+  --build-arg PARALLEL_LEVEL=$PARALLEL_LEVEL \
+  -t $JNI_DOCKER_CICD_IMAGE \
   $REPODIR/build
 
+if [ "$JNI_DOCKER_DEV_BUILD" == "ON" ]; then
+  echo "Building docker image for local development, in which gcc-toolset is enabled by default..."
+  JNI_DOCKER_DEV_IMAGE=${JNI_DOCKER_CICD_IMAGE}-devel
+  $DOCKER_CMD build -f $REPODIR/build/Dockerfile.devel \
+  --build-arg JNI_DOCKER_CICD_IMAGE=$JNI_DOCKER_CICD_IMAGE \
+  --build-arg PARALLEL_LEVEL=$PARALLEL_LEVEL \
+  -t $JNI_DOCKER_DEV_IMAGE \
+  $REPODIR/build
+  JNI_DOCKER_IMAGE=$JNI_DOCKER_DEV_IMAGE
+else
+  JNI_DOCKER_IMAGE=$JNI_DOCKER_CICD_IMAGE
+fi
+
 if [[ "$DOCKER_CMD" == "docker" ]]; then
   DOCKER_GPU_OPTS=${DOCKER_GPU_OPTS:-"--gpus all"}
 fi

ci/Dockerfile

@@ -15,7 +15,7 @@
 #
 
 ###
-# Build the image for spark-rapids-jni development environment.
+# Build the image for spark-rapids-jni CICD environment.
 #
 # Arguments: CUDA_VERSION=[12.X.Y], OS_RELEASE=[8, 9], TARGETPLATFORM=[linux/amd64, linux/arm64]
 #
@@ -28,15 +28,15 @@ ARG TARGETPLATFORM=linux/amd64
 # check available official arm-based docker images at https://hub.docker.com/r/nvidia/cuda/tags (OS/ARCH)
 FROM --platform=$TARGETPLATFORM nvidia/cuda:$CUDA_VERSION-devel-rockylinux$OS_RELEASE
 
-# If DEV_BUILD is ON, the gcc-toolset will be enabled by default for bash shell
-ARG DEV_BUILD=OFF
-
 # Dependency versions
 # Act as default GCC toolset in the image
 ARG TOOLSET_VERSION=14
 ARG CMAKE_VERSION=3.30.4
 ARG CCACHE_VERSION=4.11.2
 
+# Make it available at runtime and to the inherited images (e.g. Dockerfile.devel)
+ENV TOOLSET_VERSION=${TOOLSET_VERSION}
+
 # Default x86_64 from x86 build, aarch64 cmake for arm build
 ARG CMAKE_ARCH=x86_64
 
@@ -47,11 +47,6 @@ RUN dnf --enablerepo=powertools install -y scl-utils gcc-toolset-11 gcc-toolset-
   alternatives --set python /usr/bin/python3 && \
   python -m pip install requests 'urllib3<2.0'
 
-# Enable the gcc-toolset by default for bash shell if DEV_BUILD is ON
-RUN if [ "$DEV_BUILD" = "ON" ]; then \
-        echo "source scl_source enable gcc-toolset-${TOOLSET_VERSION}" >> /etc/bashrc; \
-    fi
-
 # Execute every time a new non-interactive bash shell is started
 ENV BASH_ENV=/etc/bashrc
 
@@ -66,6 +61,7 @@ RUN cd /usr/local && wget --quiet https://github.com/Kitware/CMake/releases/down
 RUN ln -s /usr/local/cmake-${CMAKE_VERSION}-linux-${CMAKE_ARCH}/bin/cmake /usr/local/bin/cmake
 
 # ccache for interactive builds
+ARG PARALLEL_LEVEL
 RUN cd /tmp && wget --quiet https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \
    tar zxf ccache-${CCACHE_VERSION}.tar.gz && \
    rm ccache-${CCACHE_VERSION}.tar.gz && \

src/main/cpp/benchmarks/bloom_filter.cu

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023-2024, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2026, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,17 +22,18 @@
 #include <hash/hash.hpp>
 #include <nvbench/nvbench.cuh>
 
-static void bloom_filter_put(nvbench::state& state)
+namespace {
+
+void bloom_filter_put_impl(nvbench::state& state, int version)
 {
   constexpr int num_rows   = 150'000'000;
   constexpr int num_hashes = 3;
 
-  // create the bloom filter
   cudf::size_type const bloom_filter_bytes = state.get_int64("bloom_filter_bytes");
   cudf::size_type const bloom_filter_longs = bloom_filter_bytes / sizeof(int64_t);
-  auto bloom_filter = spark_rapids_jni::bloom_filter_create(num_hashes, bloom_filter_longs);
+  auto bloom_filter =
+    spark_rapids_jni::bloom_filter_create(version, num_hashes, bloom_filter_longs);
 
-  // create a column of hashed values
   data_profile_builder builder;
   builder.no_validity();
   auto const src   = create_random_table({{cudf::type_id::INT64}}, row_count{num_rows}, builder);
@@ -41,7 +42,7 @@ static void bloom_filter_put(nvbench::state& state)
   auto const stream = cudf::get_default_stream();
   state.set_cuda_stream(nvbench::make_cuda_stream_view(stream.value()));
   state.exec(nvbench::exec_tag::timer | nvbench::exec_tag::sync,
-             [&](nvbench::launch& launch, auto& timer) {
+             [&](nvbench::launch&, auto& timer) {
                timer.start();
                spark_rapids_jni::bloom_filter_put(*bloom_filter, *input);
                stream.synchronize();
@@ -57,7 +58,24 @@ static void bloom_filter_put(nvbench::state& state)
   state.add_element_count(static_cast<double>(bytes_written) / time, "Write bytes/sec");
 }
 
-NVBENCH_BENCH(bloom_filter_put)
-  .set_name("Bloom Filter Put")
+void bloom_filter_put_v1(nvbench::state& state)
+{
+  bloom_filter_put_impl(state, spark_rapids_jni::bloom_filter_version_1);
+}
+
+void bloom_filter_put_v2(nvbench::state& state)
+{
+  bloom_filter_put_impl(state, spark_rapids_jni::bloom_filter_version_2);
+}
+
+}  // namespace
+
+NVBENCH_BENCH(bloom_filter_put_v1)
+  .set_name("Bloom Filter Put V1")
+  .add_int64_axis("bloom_filter_bytes",
+                  {512 * 1024, 1024 * 1024, 2 * 1024 * 1024, 4 * 1024 * 1024, 8 * 1024 * 1024});
+
+NVBENCH_BENCH(bloom_filter_put_v2)
+  .set_name("Bloom Filter Put V2")
   .add_int64_axis("bloom_filter_bytes",
                   {512 * 1024, 1024 * 1024, 2 * 1024 * 1024, 4 * 1024 * 1024, 8 * 1024 * 1024});

src/main/cpp/src/BloomFilterJni.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023-2025, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2026, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,17 +20,33 @@
 #include "jni_utils.hpp"
 #include "utilities.hpp"
 
+#include <limits>
+
 extern "C" {
 
 JNIEXPORT jlong JNICALL Java_com_nvidia_spark_rapids_jni_BloomFilter_creategpu(
-  JNIEnv* env, jclass, jint numHashes, jlong bloomFilterBits)
+  JNIEnv* env, jclass, jint version, jint numHashes, jlong bloomFilterBits, jint seed)
 {
   JNI_TRY
   {
     cudf::jni::auto_set_device(env);
 
-    int bloom_filter_longs = static_cast<int>((bloomFilterBits + 63) / 64);
-    auto bloom_filter      = spark_rapids_jni::bloom_filter_create(numHashes, bloom_filter_longs);
+    // Per the Spark implementation, according to the BitArray class,
+    // https://github.com/apache/spark/blob/5075ea6a85f3f1689766cf08a7d5b2ce500be1fb/common/sketch/src/main/java/org/apache/spark/util/sketch/BitArray.java#L34
+    // the number of longs representing the bit array can only be Integer.MAX_VALUE, at the most.
+    // (This is presumably because the BitArray is indexed with an int32_t.)
+    // This implies that the maximum supported bloom filter bit count is Integer.MAX_VALUE * 64.
+
+    JNI_ARG_CHECK(
+      env,
+      bloomFilterBits > 0 &&
+        bloomFilterBits <= static_cast<int64_t>(std::numeric_limits<int32_t>::max()) * 64,
+      "bloom filter bit count must be positive and less than or equal to the maximum supported "
+      "size",
+      0);
+    auto const bloom_filter_longs = static_cast<int32_t>((bloomFilterBits + 63) / 64);
+    auto bloom_filter =
+      spark_rapids_jni::bloom_filter_create(version, numHashes, bloom_filter_longs, seed);
     return reinterpret_cast<jlong>(bloom_filter.release());
   }
   JNI_CATCH(env, 0);