Skip to content

Commit 58d8f0d

Browse files
Hyper-FFHyper-FF
authored
[Enhancement] Upgrade Apache Arrow thirdparty to 24.0.0 (#75868)
1 parent 2f4ac8f commit 58d8f0d

13 files changed

Lines changed: 66 additions & 432 deletions

‎be/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -649,6 +649,7 @@ set(STARROCKS_DEPENDENCIES
649649
zstd
650650
streamvbyte
651651
arrow
652+
arrow_testing
652653
arrow_flight
653654
arrow_flight_sql
654655
gRPC::grpc

‎be/src/formats/parquet/level_builder.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,8 +394,8 @@ Status LevelBuilder::_write_datetime_column_chunk(const LevelBuilderContext& ctx
394394
: data_col[i]._timestamp;
395395

396396
if constexpr (use_int96_timestamp_encoding) {
397-
auto date = reinterpret_cast<int32_t*>(values[i].value + 2);
398-
auto nanosecond = reinterpret_cast<int64_t*>(values[i].value);
397+
auto date = reinterpret_cast<int32_t*>(values[i].value.data() + 2);
398+
auto nanosecond = reinterpret_cast<int64_t*>(values[i].value.data());
399399
*date = timestamp::to_julian(timestamp);
400400
*nanosecond = timestamp::to_time(timestamp) * 1000;
401401
} else {

‎be/test/column/column_to_arrow_converter_test.cpp‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@ DIAGNOSTIC_IGNORE("-Wclass-memaccess")
4242
DIAGNOSTIC_POP
4343
#endif
4444

45-
#include <arrow/ipc/json_simple.h>
4645
#include <arrow/memory_pool.h>
4746
#include <arrow/pretty_print.h>
47+
#include <arrow/testing/gtest_util.h>
4848

4949
#include "base/hash/unaligned_access.h"
5050
#include "base/types/int128.h"
@@ -528,9 +528,8 @@ TEST_F(StarRocksColumnToArrowTest, testArrayColumn) {
528528
std::shared_ptr<arrow::Array> array;
529529
convert_to_arrow(array_type_desc, column, arrow_type, memory_pool.get(), &array);
530530

531-
auto s = arrow::ipc::internal::json::ArrayFromJSON(arrow_type, "[[1, 2, 3], [4, null, 5, 6], [], [null, null]]");
532-
ASSERT_TRUE(s.ok());
533-
ASSERT_TRUE(s.ValueUnsafe()->Equals(array));
531+
auto s = arrow::ArrayFromJSON(arrow_type, "[[1, 2, 3], [4, null, 5, 6], [], [null, null]]");
532+
ASSERT_TRUE(s->Equals(array));
534533
}
535534

536535
TEST_F(StarRocksColumnToArrowTest, testNullableArrayColumn) {
@@ -554,10 +553,8 @@ TEST_F(StarRocksColumnToArrowTest, testNullableArrayColumn) {
554553
convert_to_arrow(array_type_desc, column, arrow_type, memory_pool.get(), &array);
555554

556555
std::shared_ptr<arrow::Array> expect_array;
557-
auto s = arrow::ipc::internal::json::ArrayFromJSON(arrow_type,
558-
"[[1, 2, 3], null, [4, null, 5, 6], [], [null, null]]");
559-
ASSERT_TRUE(s.ok());
560-
ASSERT_TRUE(s.ValueUnsafe()->Equals(array));
556+
auto s = arrow::ArrayFromJSON(arrow_type, "[[1, 2, 3], null, [4, null, 5, 6], [], [null, null]]");
557+
ASSERT_TRUE(s->Equals(array));
561558
}
562559

563560
TEST_F(StarRocksColumnToArrowTest, testStructColumn) {
@@ -581,14 +578,13 @@ TEST_F(StarRocksColumnToArrowTest, testStructColumn) {
581578
std::shared_ptr<arrow::Array> array;
582579
convert_to_arrow(struct_type_desc, column, arrow_type, memory_pool.get(), &array);
583580

584-
auto s = arrow::ipc::internal::json::ArrayFromJSON(arrow_type, R"([
581+
auto s = arrow::ArrayFromJSON(arrow_type, R"([
585582
{"id": 1, "name": "test1"},
586583
{"id": null, "name": "test2"},
587584
{"id": 2, "name": null},
588585
{"id": null, "name": null}
589586
])");
590-
ASSERT_TRUE(s.ok());
591-
ASSERT_TRUE(s.ValueUnsafe()->Equals(array));
587+
ASSERT_TRUE(s->Equals(array));
592588
}
593589

594590
TEST_F(StarRocksColumnToArrowTest, testNullableStructColumn) {
@@ -615,15 +611,14 @@ TEST_F(StarRocksColumnToArrowTest, testNullableStructColumn) {
615611
convert_to_arrow(struct_type_desc, column, arrow_type, memory_pool.get(), &array);
616612

617613
std::shared_ptr<arrow::Array> expect_array;
618-
auto s = arrow::ipc::internal::json::ArrayFromJSON(arrow_type, R"([
614+
auto s = arrow::ArrayFromJSON(arrow_type, R"([
619615
{"id": 1, "name": "test1"},
620616
null,
621617
{"id": null, "name": "test2"},
622618
{"id": 2, "name": null},
623619
{"id": null, "name": null}
624620
])");
625-
ASSERT_TRUE(s.ok());
626-
ASSERT_TRUE(s.ValueUnsafe()->Equals(array));
621+
ASSERT_TRUE(s->Equals(array));
627622
}
628623

629624
TEST_F(StarRocksColumnToArrowTest, testMapColumn) {

‎nix/thirdparty-archives.nix‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ let
99
md5 = "955b6faedf32ec2ce1b7725561d15618";
1010
sha256 = "00nnmg2mazx8i1brxzvbmfq5j7738md884373nx8jq0096di4222";
1111
};
12-
"arrow-apache-arrow-19.0.1.tar.gz" = {
13-
url = "https://github.com/apache/arrow/archive/refs/tags/apache-arrow-19.0.1.tar.gz";
14-
md5 = "8c5091da0f8fb41a47d7f4dad7b712df";
15-
sha256 = "0snqca1wlkxc22n8k2gs21g6py8r575hwwgqns3cqhc8jl28b2ac";
12+
"arrow-apache-arrow-24.0.0.tar.gz" = {
13+
url = "https://github.com/apache/arrow/archive/refs/tags/apache-arrow-24.0.0.tar.gz";
14+
md5 = "66c53bd00baa79034bd2ca167beea436";
15+
sha256 = "0a3b43avih2kiniav2rhp24awdhmxhzp4fmvlr6lqci6iwc8vqcl";
1616
};
1717
"avro-release-1.12.0.tar.gz" = {
1818
url = "https://github.com/apache/avro/archive/refs/tags/release-1.12.0.tar.gz";
@@ -349,10 +349,10 @@ let
349349
md5 = "161cbf4c347f6daadacfb749c31842f8";
350350
sha256 = "1g943v85ipgyxb62hrn54fsc5aicihrp6rwvj0pb6p19l181swym";
351351
};
352-
"xsimd-13.2.0.tar.gz" = {
353-
url = "https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.2.0.tar.gz";
354-
md5 = "f451a1c57d2a4fdc0ba663be438dced4";
355-
sha256 = "0yj4hbdfrhwkdny6fb7yq5jbw6pldp1m671jf3f5l64cahywvn7d";
352+
"xsimd-14.0.0.tar.gz" = {
353+
url = "https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz";
354+
md5 = "75c0d34cf7011924ba19978076c76dc1";
355+
sha256 = "1khd94aipz09blfxvnp53g9r4frsbx68v4yn146c2ma9jlv05php";
356356
};
357357
"xxHash-0.8.3.tar.gz" = {
358358
url = "https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.3.tar.gz";
@@ -365,9 +365,9 @@ let
365365
sha256 = "1l87r6h8qccba1yjx6n13q36yh0wrq3rzz9nnraq5ssjr2lmmipr";
366366
};
367367
"zstd-1.5.7.tar.gz" = {
368-
url = "https://github.com/facebook/zstd/archive/v1.5.7.tar.gz";
369-
md5 = "619a019adbbc4536e7fb93cdbb01af3e";
370-
sha256 = "1qrf074w0wlykrj3pgfs5qp90s374a05pa0wdvjm82djar2jimrp";
368+
url = "https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz";
369+
md5 = "780fc1896922b1bc52a4e90980cdda48";
370+
sha256 = "18vgkvh7w6zw4jn2aj1mp0yv98m4fk52ay6da0wh4pm194gyaczb";
371371
};
372372
};
373373

@@ -486,7 +486,7 @@ let
486486
"librdkafka-2.11.0.tar.gz"
487487
"pulsar-client-3.3.0.tar.gz"
488488
"flatbuffers-v1.10.0.tar.gz"
489-
"arrow-apache-arrow-19.0.1.tar.gz"
489+
"arrow-apache-arrow-24.0.0.tar.gz"
490490
"brotli-1.0.9.tar.gz"
491491
"zstd-1.5.7.tar.gz"
492492
"s2geometry-0.9.0.tar.gz"
@@ -521,7 +521,7 @@ let
521521
"simdutf-5.2.8.tar.gz"
522522
"poco-1.12.5-release.tar.gz"
523523
"icu4c-76_1-src.zip"
524-
"xsimd-13.2.0.tar.gz"
524+
"xsimd-14.0.0.tar.gz"
525525
"libxml2-2.11.7.tar.gz"
526526
"azure-storage-files-shares_12.12.0.tar.gz"
527527
"libdivide-v5.2.0.tar.gz"
@@ -557,7 +557,7 @@ let
557557
"librdkafka-2.11.0.tar.gz"
558558
"pulsar-client-3.3.0.tar.gz"
559559
"flatbuffers-v1.10.0.tar.gz"
560-
"arrow-apache-arrow-19.0.1.tar.gz"
560+
"arrow-apache-arrow-24.0.0.tar.gz"
561561
"brotli-1.0.9.tar.gz"
562562
"zstd-1.5.7.tar.gz"
563563
"s2geometry-0.9.0.tar.gz"
@@ -590,7 +590,7 @@ let
590590
"simdutf-5.2.8.tar.gz"
591591
"poco-1.12.5-release.tar.gz"
592592
"icu4c-76_1-src.zip"
593-
"xsimd-13.2.0.tar.gz"
593+
"xsimd-14.0.0.tar.gz"
594594
"libxml2-2.11.7.tar.gz"
595595
"azure-storage-files-shares_12.12.0.tar.gz"
596596
"libdivide-v5.2.0.tar.gz"
@@ -626,7 +626,7 @@ let
626626
"librdkafka-2.11.0.tar.gz"
627627
"pulsar-client-3.3.0.tar.gz"
628628
"flatbuffers-v1.10.0.tar.gz"
629-
"arrow-apache-arrow-19.0.1.tar.gz"
629+
"arrow-apache-arrow-24.0.0.tar.gz"
630630
"brotli-1.0.9.tar.gz"
631631
"zstd-1.5.7.tar.gz"
632632
"s2geometry-0.9.0.tar.gz"
@@ -659,7 +659,7 @@ let
659659
"simdutf-5.2.8.tar.gz"
660660
"poco-1.12.5-release.tar.gz"
661661
"icu4c-76_1-src.zip"
662-
"xsimd-13.2.0.tar.gz"
662+
"xsimd-14.0.0.tar.gz"
663663
"libxml2-2.11.7.tar.gz"
664664
"azure-storage-files-shares_12.12.0.tar.gz"
665665
"libdivide-v5.2.0.tar.gz"

‎thirdparty/build-thirdparty-darwin.sh‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2019,6 +2019,10 @@ build_arrow() {
20192019
-DARROW_BUILD_STATIC=ON \
20202020
-DARROW_BUILD_SHARED=OFF \
20212021
-DARROW_BUILD_TESTS=OFF \
2022+
-DARROW_TESTING=ON \
2023+
-DGTest_SOURCE=SYSTEM \
2024+
-DGTest_ROOT="${TP_INSTALL_DIR}" \
2025+
-DCMAKE_CXX_FLAGS="${CXXFLAGS} -fno-sized-deallocation" \
20222026
-DARROW_BUILD_EXAMPLES=OFF \
20232027
-DARROW_BUILD_INTEGRATION=OFF \
20242028
-DARROW_BUILD_UTILITIES=OFF \
@@ -2082,7 +2086,8 @@ build_arrow() {
20822086
-Dflatbuffers_ROOT="${flatbuffers_prefix}" \
20832087
-DCMAKE_PREFIX_PATH="${arrow_prefix_path}" \
20842088
-DThrift_ROOT="${TP_INSTALL_DIR}" \
2085-
-Dthrift_SOURCE=SYSTEM \
2089+
-DARROW_THRIFT_USE_SHARED=OFF \
2090+
-DThrift_SOURCE=SYSTEM \
20862091
-Dxsimd_SOURCE=SYSTEM \
20872092
-Dxsimd_DIR="${TP_INSTALL_DIR}/share/cmake/xsimd" \
20882093
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
@@ -2908,10 +2913,12 @@ build_benchgen() {
29082913
cd "${TP_SOURCE_DIR}/${BENCHGEN_SOURCE}"
29092914
perl -0pi -e 's/brotlicommon snappy zstd\)/brotlicommon lz4 snappy zstd)/' \
29102915
cmake_modules/BenchmarkArrow.cmake
2916+
perl -0pi -e 's/set\(CMAKE_CXX_STANDARD 17\)/set(CMAKE_CXX_STANDARD 20)/' CMakeLists.txt
29112917
"${CMAKE_CMD}" -G "${CMAKE_GENERATOR}" \
29122918
-DCMAKE_BUILD_TYPE=Release \
29132919
-DCMAKE_INSTALL_LIBDIR=lib \
29142920
-DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
2921+
-DCMAKE_CXX_FLAGS="${CXXFLAGS} -fno-sized-deallocation" \
29152922
-DBENCHGEN_ARROW_PREFIX="${TP_INSTALL_DIR}" \
29162923
-S . -B build
29172924
"${CMAKE_CMD}" --build build -j "${PARALLEL}"

‎thirdparty/build-thirdparty.sh‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -895,7 +895,7 @@ build_brotli() {
895895

896896
# arrow
897897
build_arrow() {
898-
export CXXFLAGS="-O3 -fno-omit-frame-pointer -fPIC -g ${FILE_PREFIX_MAP_OPTION}"
898+
export CXXFLAGS="-O3 -fno-omit-frame-pointer -fPIC -g -fno-sized-deallocation ${FILE_PREFIX_MAP_OPTION}"
899899
export CFLAGS="-O3 -fno-omit-frame-pointer -fPIC -g ${FILE_PREFIX_MAP_OPTION}"
900900
export CPPFLAGS=$CXXFLAGS
901901

@@ -931,7 +931,7 @@ build_arrow() {
931931
# so disable jemalloc here and use SystemAllocator.
932932
#
933933
# Currently, the standard APIs are hooked in BE, so the jemalloc standard APIs will actually be used.
934-
${CMAKE_CMD} -DARROW_PARQUET=ON -DARROW_JSON=ON -DARROW_IPC=ON -DARROW_USE_GLOG=OFF -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF \
934+
${CMAKE_CMD} -DARROW_TESTING=ON -DGTest_SOURCE=SYSTEM -DGTest_ROOT=$TP_INSTALL_DIR -DARROW_PARQUET=ON -DARROW_JSON=ON -DARROW_IPC=ON -DARROW_USE_GLOG=OFF -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF \
935935
-DARROW_WITH_BROTLI=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON \
936936
-DARROW_WITH_UTF8PROC=OFF -DARROW_WITH_RE2=OFF \
937937
-DARROW_JEMALLOC=OFF -DARROW_MIMALLOC=OFF \
@@ -964,7 +964,8 @@ build_arrow() {
964964
-DCMAKE_PREFIX_PATH=${TP_INSTALL_DIR} \
965965
-G "${CMAKE_GENERATOR}" \
966966
-DThrift_ROOT=$TP_INSTALL_DIR/ \
967-
-Dthrift_SOURCE=SYSTEM \
967+
-DARROW_THRIFT_USE_SHARED=OFF \
968+
-DThrift_SOURCE=SYSTEM \
968969
-Dxsimd_SOURCE=SYSTEM \
969970
-Dxsimd_DIR=$TP_INSTALL_DIR/share/cmake/xsimd ..
970971

@@ -1726,9 +1727,12 @@ build_benchgen() {
17261727
cd ${TP_SOURCE_DIR}/${BENCHGEN_SOURCE}
17271728
perl -0pi -e 's/brotlicommon snappy zstd\)/brotlicommon lz4 snappy zstd)/' \
17281729
cmake_modules/BenchmarkArrow.cmake
1730+
perl -0pi -e 's/set\(CMAKE_CXX_STANDARD 17\)/set(CMAKE_CXX_STANDARD 20)/' CMakeLists.txt
17291731
${CMAKE_CMD} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Release \
17301732
-DCMAKE_INSTALL_LIBDIR=lib \
17311733
-DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
1734+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
1735+
-DCMAKE_CXX_FLAGS="${CXXFLAGS} -fno-sized-deallocation" \
17321736
-DBENCHGEN_ARROW_PREFIX="${TP_INSTALL_DIR}" -S . -B build
17331737
${CMAKE_CMD} --build build -j "${PARALLEL}"
17341738
${CMAKE_CMD} --install build

‎thirdparty/download-thirdparty.sh‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -712,12 +712,12 @@ cd -
712712
# patch arrow
713713
if [[ -d $TP_SOURCE_DIR/$ARROW_SOURCE ]] ; then
714714
cd $TP_SOURCE_DIR/$ARROW_SOURCE
715-
if [ ! -f $PATCHED_MARK ] && [ $ARROW_SOURCE = "arrow-apache-arrow-19.0.1" ] ; then
716-
patch -p1 < $TP_PATCH_DIR/arrow-19.0.1-parquet-map-key.patch
717-
patch -p1 < $TP_PATCH_DIR/arrow-19.0.1-use-zstd-1.5.7.patch
718-
patch -p1 < $TP_PATCH_DIR/arrow-19.0.1-flight-types-clang.patch
719-
patch -p1 < $TP_PATCH_DIR/arrow-19.0.1-libtool-version-check.patch
720-
patch -p1 < $TP_PATCH_DIR/arrow-19.0.1-thrift.patch
715+
if [ ! -f $PATCHED_MARK ] && [ $ARROW_SOURCE = "arrow-apache-arrow-24.0.0" ] ; then
716+
# Allow reading parquet files with Hive-style optional MAP keys.
717+
# The other arrow-19.0.1 patches are obsolete on 24.0.0: zstd 1.5.7 is
718+
# already upstream; thrift is provided via the system build (-DThrift_SOURCE=SYSTEM);
719+
# the flight-types reorder and the macOS libtool-version check are fixed upstream.
720+
patch -p1 < $TP_PATCH_DIR/arrow-24.0.0-parquet-map-key.patch
721721
touch $PATCHED_MARK
722722
fi
723723
cd -

0 commit comments

Comments
 (0)