Back out "build: Update bundled Apache Arrow to 21.0.0 from 15.0.0" (facebookincubator#14563)

Summary:

This fixes issue : facebookincubator#14562

Original commit changeset: 517200e90c9c

Original Phabricator Diff: D80488815

Differential Revision: D80741829

Author: Krishna Pai

CMake/FindArrow.cmake

@@ -14,30 +14,30 @@
 
 include(FindPackageHandleStandardArgs)
 
-set(find_package_args)
-if(Arrow_FIND_VERSION)
-  list(APPEND find_package_args ${Arrow_FIND_VERSION})
-endif()
-if(Arrow_FIND_QUIETLY)
-  list(APPEND find_package_args QUIET)
-endif()
-find_package(Arrow ${find_package_args} CONFIG)
-if(Arrow_VERSION VERSION_LESS_EQUAL 21.0.0)
-  # Workaround for https://github.com/apache/arrow/issues/46386 .
-  # ArrowTestingConfig.cmake may call this file via find_dependency(Arrow). It
-  # causes an infinite loop.
-  set(CMAKE_FIND_PACKAGE_PREFER_CONFIG_KEEP ${CMAKE_FIND_PACKAGE_PREFER_CONFIG})
-  set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE)
-endif()
-find_package(ArrowTesting ${find_package_args} CONFIG)
-if(Arrow_VERSION VERSION_LESS_EQUAL 21.0.0)
-  set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ${CMAKE_FIND_PACKAGE_PREFER_CONFIG_KEEP})
-endif()
+find_library(ARROW_LIB libarrow.a)
+find_library(ARROW_TESTING_LIB libarrow_testing.a)
+find_path(ARROW_INCLUDE_PATH arrow/api.h)
+find_package(Thrift)
+
+find_package_handle_standard_args(
+  Arrow
+  DEFAULT_MSG
+  ARROW_LIB
+  ARROW_TESTING_LIB
+  ARROW_INCLUDE_PATH
+  Thrift_FOUND)
 
 # Only add the libraries once.
-if(Arrow_FOUND
-   AND ArrowTesting_FOUND
-   AND NOT TARGET arrow)
-  add_library(arrow ALIAS Arrow::arrow_static)
-  add_library(arrow_testing ALIAS ArrowTesting::arrow_testing_static)
+if(Arrow_FOUND AND NOT TARGET arrow)
+  add_library(arrow STATIC IMPORTED GLOBAL)
+  add_library(arrow_testing STATIC IMPORTED GLOBAL)
+  add_library(thrift ALIAS thrift::thrift)
+
+  set_target_properties(
+    arrow arrow_testing PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                                   ${ARROW_INCLUDE_PATH})
+  set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${ARROW_LIB}
+                                         INTERFACE_LINK_LIBRARIES thrift)
+  set_target_properties(arrow_testing PROPERTIES IMPORTED_LOCATION
+                                                 ${ARROW_TESTING_LIB})
 endif()

CMake/resolve_dependency_modules/arrow/CMakeLists.txt

@@ -19,13 +19,6 @@ if(VELOX_ENABLE_ARROW)
   else()
     set(THRIFT_SOURCE "BUNDLED")
   endif()
-  # Apache Arrow 21.0.0 uses archive.apache.org as the default mirror, which
-  # tends to timeout in CI. We don't need to explicitly set this alternate
-  # mirror after the 22.0.0 release because this will be used by default. See
-  # also: https://github.com/apache/arrow/issues/47111
-  set(THRIFT_URL
-      "https://packages.apache.org/artifactory/arrow/thirdparty/thrift-0.22.0.tar.gz"
-  )
 
   set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep")
   set(ARROW_CMAKE_ARGS
@@ -37,39 +30,38 @@ if(VELOX_ENABLE_ARROW)
       -DARROW_WITH_ZLIB=ON
       -DARROW_WITH_ZSTD=ON
       -DARROW_JEMALLOC=OFF
-      -DARROW_MIMALLOC=OFF
       -DARROW_SIMD_LEVEL=NONE
       -DARROW_RUNTIME_SIMD_LEVEL=NONE
       -DARROW_WITH_UTF8PROC=OFF
       -DARROW_TESTING=ON
       -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}/install
       -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-      -DCMAKE_DEBUG_POSTFIX=
       -DARROW_BUILD_STATIC=ON
       -DThrift_SOURCE=${THRIFT_SOURCE}
-      -DARROW_THRIFT_BUILD_SHA256_CHECKSUM=${THRIFT_BUILD_SHA256_CHECKSUM}
-      -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH})
+      -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}
+      -DCMAKE_POLICY_VERSION_MINIMUM=3.5
+      # Remove with Arrow upgrade to Arrow 20.
+      -DARROW_CXXFLAGS=-Wno-documentation)
   set(ARROW_LIBDIR ${ARROW_PREFIX}/install/${CMAKE_INSTALL_LIBDIR})
 
   add_library(thrift STATIC IMPORTED GLOBAL)
   if(NOT Thrift_FOUND)
-    set(THRIFT_SOURCE_ROOT ${ARROW_PREFIX}/src/arrow_ep-build/_deps/thrift-src)
-    set(THRIFT_BUILD_ROOT ${ARROW_PREFIX}/src/arrow_ep-build/_deps/thrift-build)
-    set(THRIFT_INCLUDE_DIR ${THRIFT_BUILD_ROOT}
-                           ${THRIFT_SOURCE_ROOT}/lib/cpp/src)
-    foreach(dir ${THRIFT_INCLUDE_DIR})
-      file(MAKE_DIRECTORY ${dir})
-    endforeach()
-    set(THRIFT_LIB ${ARROW_PREFIX}/src/arrow_ep-build/lib/libthrift.a)
+    set(THRIFT_ROOT ${ARROW_PREFIX}/src/arrow_ep-build/thrift_ep-install)
+    set(THRIFT_LIB ${THRIFT_ROOT}/lib/libthrift.a)
+
+    file(MAKE_DIRECTORY ${THRIFT_ROOT}/include)
+    set(THRIFT_INCLUDE_DIR ${THRIFT_ROOT}/include)
   endif()
-  target_include_directories(thrift INTERFACE ${THRIFT_INCLUDE_DIR})
+
+  set_property(TARGET thrift PROPERTY INTERFACE_INCLUDE_DIRECTORIES
+                                      ${THRIFT_INCLUDE_DIR})
   set_property(TARGET thrift PROPERTY IMPORTED_LOCATION ${THRIFT_LIB})
 
-  set(VELOX_ARROW_BUILD_VERSION 21.0.0)
+  set(VELOX_ARROW_BUILD_VERSION 15.0.0)
   set(VELOX_ARROW_BUILD_SHA256_CHECKSUM
-      5d3f8db7e72fb9f65f4785b7a1634522e8d8e9657a445af53d4a34a3849857b5)
+      ab74c60c46938505c8cd7599b1d2826c68450645d5860d0ff40f67e371a5d0b5)
   set(VELOX_ARROW_SOURCE_URL
-      "https://github.com/apache/arrow/releases/download/apache-arrow-${VELOX_ARROW_BUILD_VERSION}/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz"
+      "https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz"
   )
 
   velox_resolve_dependency_url(ARROW)
@@ -80,19 +72,11 @@ if(VELOX_ENABLE_ARROW)
     URL ${VELOX_ARROW_SOURCE_URL}
     URL_HASH ${VELOX_ARROW_BUILD_SHA256_CHECKSUM}
     SOURCE_SUBDIR cpp
-    # We can't change ARROW_THRIFT_URL by CMake variable. We need to use
-    # environment variable. So we use "cmake -E env ARROW_THRIFT_URL=..." here.
-    # We can remove this when we use Apache Arrow C++ 22.0.0 or later. See also
-    # the above comment THRIFT_URL.
-    CMAKE_COMMAND ${CMAKE_COMMAND}
-    -E
-    env
-    ARROW_THRIFT_URL=${THRIFT_URL}
-    --
-    ${CMAKE_COMMAND}
     CMAKE_ARGS ${ARROW_CMAKE_ARGS}
     BUILD_BYPRODUCTS ${ARROW_LIBDIR}/libarrow.a
-                     ${ARROW_LIBDIR}/libarrow_testing.a ${THRIFT_LIB})
+                     ${ARROW_LIBDIR}/libarrow_testing.a ${THRIFT_LIB}
+    PATCH_COMMAND git apply ${CMAKE_CURRENT_LIST_DIR}/thrift-download.patch &&
+                  git apply ${CMAKE_CURRENT_LIST_DIR}/cmake-compatibility.patch)
 
   add_library(arrow STATIC IMPORTED GLOBAL)
   add_library(arrow_testing STATIC IMPORTED GLOBAL)
@@ -108,4 +92,5 @@ if(VELOX_ENABLE_ARROW)
   set_target_properties(
     arrow_testing PROPERTIES IMPORTED_LOCATION
                              ${ARROW_LIBDIR}/libarrow_testing.a)
+
 endif()

CMake/resolve_dependency_modules/arrow/cmake-compatibility.patch

@@ -0,0 +1,34 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+@@ -971,7 +971,8 @@
+     -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=${CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY}
+     -DCMAKE_INSTALL_LIBDIR=lib
+     -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT}
+-    -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE})
++    -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE}
++    -DCMAKE_POLICY_VERSION_MINIMUM=3.5)
+
+ # Enable s/ccache if set by parent.
+ if(CMAKE_C_COMPILER_LAUNCHER AND CMAKE_CXX_COMPILER_LAUNCHER)
+@@ -1026,6 +1027,7 @@
+   set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE)
+   set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY TRUE)
+   set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH})
++  set(CMAKE_POLICY_VERSION_MINIMUM 3.5)
+   if(MSVC)
+     string(REPLACE "/WX" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
+     string(REPLACE "/WX" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")

CMake/resolve_dependency_modules/arrow/thrift-download.patch

@@ -0,0 +1,58 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--- a/cpp/thirdparty/versions.txt
++++ b/cpp/thirdparty/versions.txt
+@@ -108,7 +108,7 @@
+ ARROW_S2N_TLS_BUILD_VERSION=v1.3.35
+ ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=9d32b26e6bfcc058d98248bf8fc231537e347395dd89cf62bb432b55c5da990d
+ ARROW_THRIFT_BUILD_VERSION=0.16.0
+-ARROW_THRIFT_BUILD_SHA256_CHECKSUM=f460b5c1ca30d8918ff95ea3eb6291b3951cf518553566088f3f2be8981f6209
++ARROW_THRIFT_BUILD_SHA256_CHECKSUM=df2931de646a366c2e5962af679018bca2395d586e00ba82d09c0379f14f8e7b
+ ARROW_UCX_BUILD_VERSION=1.12.1
+ ARROW_UCX_BUILD_SHA256_CHECKSUM=9bef31aed0e28bf1973d28d74d9ac4f8926c43ca3b7010bd22a084e164e31b71
+ ARROW_UTF8PROC_BUILD_VERSION=v2.7.0
+@@ -164,7 +164,7 @@
+   "ARROW_RE2_URL re2-${ARROW_RE2_BUILD_VERSION}.tar.gz https://github.com/google/re2/archive/${ARROW_RE2_BUILD_VERSION}.tar.gz"
+   "ARROW_S2N_TLS_URL s2n-${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz https://github.com/aws/s2n-tls/archive/${ARROW_S2N_TLS_BUILD_VERSION}.tar.gz"
+   "ARROW_SNAPPY_URL snappy-${ARROW_SNAPPY_BUILD_VERSION}.tar.gz https://github.com/google/snappy/archive/${ARROW_SNAPPY_BUILD_VERSION}.tar.gz"
+-  "ARROW_THRIFT_URL thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz https://archive.apache.org/dist/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
++  "ARROW_THRIFT_URL thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz https://github.com/apache/thrift/archive/refs/tags/v${ARROW_THRIFT_BUILD_VERSION}.tar.gz
+   "ARROW_UCX_URL ucx-${ARROW_UCX_BUILD_VERSION}.tar.gz https://github.com/openucx/ucx/archive/v${ARROW_UCX_BUILD_VERSION}.tar.gz"
+   "ARROW_UTF8PROC_URL utf8proc-${ARROW_UTF8PROC_BUILD_VERSION}.tar.gz https://github.com/JuliaStrings/utf8proc/archive/${ARROW_UTF8PROC_BUILD_VERSION}.tar.gz"
+   "ARROW_XSIMD_URL xsimd-${ARROW_XSIMD_BUILD_VERSION}.tar.gz https://github.com/xtensor-stack/xsimd/archive/${ARROW_XSIMD_BUILD_VERSION}.tar.gz"
+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+@@ -809,20 +809,7 @@
+   set(THRIFT_SOURCE_URL "$ENV{ARROW_THRIFT_URL}")
+ else()
+   set_urls(THRIFT_SOURCE_URL
+-           "https://www.apache.org/dyn/closer.cgi?action=download&filename=/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://downloads.apache.org/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://apache.claz.org/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://apache.cs.utah.edu/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://apache.mirrors.lucidnetworks.net/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://apache.osuosl.org/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://ftp.wayne.edu/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://mirror.olnevhost.net/pub/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://mirrors.gigenet.com/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://mirrors.koehn.com/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://mirrors.ocf.berkeley.edu/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://mirrors.sonic.net/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "https://us.mirrors.quenda.co/apache/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+-           "${THIRDPARTY_MIRROR_URL}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz")
++           "https://github.com/apache/thrift/archive/refs/tags/v${ARROW_THRIFT_BUILD_VERSION}.tar.gz")
+ endif()
+ 
+ if(DEFINED ENV{ARROW_UCX_URL})

NOTICE.txt

@@ -13,9 +13,6 @@ This product includes software from HowardHinnant's date library (MIT License).
 This product includes software from the Arrow project.
 * https://github.com/apache/arrow/blob/apache-arrow-15.0.0/cpp/src/arrow/io/hdfs_internal.h
 * https://github.com/apache/arrow/blob/apache-arrow-15.0.0/cpp/src/arrow/io/hdfs_internal.cc
-* https://github.com/apache/arrow/blob/apache-arrow-15.0.0/cpp/src/arrow/util/bpacking.h
-* https://github.com/apache/arrow/blob/apache-arrow-15.0.0/cpp/src/arrow/util/spaced.h
-
 Which contain the following NOTICE file:
 -------
   Apache Arrow

scripts/docker/centos-multi.dockerfile

@@ -28,6 +28,7 @@ COPY scripts/setup-helper-functions.sh /
 COPY scripts/setup-versions.sh /
 COPY scripts/setup-common.sh /
 COPY scripts/setup-centos9.sh /
+COPY CMake/resolve_dependency_modules/arrow/cmake-compatibility.patch /
 
 ARG VELOX_BUILD_SHARED=ON
 # Building libvelox.so requires folly and gflags to be built shared as well for now
@@ -42,7 +43,8 @@ ENV UV_TOOL_BIN_DIR=/usr/local/bin \
     INSTALL_PREFIX=/deps
 
 # CMake 4.0 removed support for cmake minimums of <=3.5 and will fail builds, this overrides it
-ENV CMAKE_POLICY_VERSION_MINIMUM="3.5"
+ENV CMAKE_POLICY_VERSION_MINIMUM="3.5" \
+    VELOX_ARROW_CMAKE_PATCH=/cmake-compatibility.patch
 
 # Some CMake configs contain the hard coded prefix '/deps', we need to replace that with
 # the future location to avoid build errors in the base-image

scripts/docker/ubuntu-22.04-cpp.dockerfile

@@ -22,8 +22,10 @@ RUN apt update && \
 
 
 COPY scripts /velox/scripts/
+COPY CMake/resolve_dependency_modules/arrow/cmake-compatibility.patch /
 
-ENV UV_TOOL_BIN_DIR=/usr/local/bin \
+ENV VELOX_ARROW_CMAKE_PATCH=/cmake-compatibility.patch \
+    UV_TOOL_BIN_DIR=/usr/local/bin \
     UV_INSTALL_DIR=/usr/local/bin
 
 # TZ and DEBIAN_FRONTEND="noninteractive"

scripts/setup-common.sh

@@ -170,6 +170,18 @@ function install_simdjson {
 
 function install_arrow {
   wget_and_untar https://github.com/apache/arrow/archive/apache-arrow-"${ARROW_VERSION}".tar.gz arrow
+  (
+    # Can be removed after an upgrade to Arrow 20.0.0
+    if [ -z "$VELOX_ARROW_CMAKE_PATCH" ]; then
+      # We need to set a different path when building the Dockerfile.
+      ABSOLUTE_SCRIPTDIR=$(realpath "$SCRIPT_DIR")
+      VELOX_ARROW_CMAKE_PATCH="$ABSOLUTE_SCRIPTDIR/../CMake/resolve_dependency_modules/arrow/cmake-compatibility.patch"
+    fi
+
+    cd "$DEPENDENCY_DIR"/arrow || exit 1
+    git apply "$VELOX_ARROW_CMAKE_PATCH"
+  ) || exit 1
+
   cmake_install_dir arrow/cpp \
     -DARROW_PARQUET=OFF \
     -DARROW_WITH_THRIFT=ON \
@@ -178,7 +190,6 @@ function install_arrow {
     -DARROW_WITH_ZLIB=ON \
     -DARROW_WITH_ZSTD=ON \
     -DARROW_JEMALLOC=OFF \
-    -DARROW_MIMALLOC=OFF \
     -DARROW_SIMD_LEVEL=NONE \
     -DARROW_RUNTIME_SIMD_LEVEL=NONE \
     -DARROW_WITH_UTF8PROC=OFF \

scripts/setup-versions.sh

@@ -22,7 +22,7 @@
 FB_OS_VERSION="v2025.04.28.00"
 FMT_VERSION="10.1.1"
 BOOST_VERSION="boost-1.84.0"
-ARROW_VERSION="21.0.0"
+ARROW_VERSION="15.0.0"
 DUCKDB_VERSION="v0.8.1"
 PROTOBUF_VERSION="21.8"
 XSIMD_VERSION="10.0.0"
@@ -35,7 +35,7 @@ GFLAGS_VERSION="v2.2.2"
 GLOG_VERSION="v0.6.0"
 LZO_VERSION="2.10"
 SNAPPY_VERSION="1.1.8"
-THRIFT_VERSION="${THRIFT_VERSION:-v0.22.0}"
+THRIFT_VERSION="${THRIFT_VERSION:-v0.16.0}"
 STEMMER_VERSION="2.2.0"
 GEOS_VERSION="3.10.7"
 # shellcheck disable=SC2034