Skip to content

Commit 6a5b057

Browse files
Hyper-FFHyper-FF
committed
[Enhancement] Upgrade Apache Arrow thirdparty to 24.0.0
Bump the C++ thirdparty Arrow from 19.0.1 to 24.0.0. Three dependency adjustments are required for Arrow 24 to build against StarRocks thirdparty: - Arrow 24 requires xsimd >= 14.0.0; bump xsimd 13.2.0 -> 14.0.0. - Arrow links each dependency as shared by default; StarRocks builds thrift static-only (libthrift.a), so FindThriftAlt could not locate the library and fell back to the bundled thrift (0.22.0), whose pinned checksum does not match StarRocks thrift 0.23.0. Add -DARROW_THRIFT_USE_SHARED=OFF (and use the correctly-cased -DThrift_SOURCE=SYSTEM) so Arrow uses the system thrift. - Arrow 24 pins zstd 1.5.7 to the official release tarball (sha256 eb33e51f...); StarRocks fed Arrow its GitHub archive tarball of the same version (different bytes), failing Arrow's bundled-zstd checksum. Switch ZSTD_DOWNLOAD to the release tarball so it matches. Verified: arrow 24.0.0 configures and compiles (libarrow.a, libparquet.a) against the system deps (thrift 0.23.0, xsimd 14.0.0, bundled zstd 1.5.7).
1 parent f7a5783 commit 6a5b057

3 files changed

Lines changed: 29 additions & 28 deletions

File tree

‎nix/thirdparty-archives.nix‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ let
99
md5 = "955b6faedf32ec2ce1b7725561d15618";
1010
sha256 = "00nnmg2mazx8i1brxzvbmfq5j7738md884373nx8jq0096di4222";
1111
};
12-
"arrow-apache-arrow-19.0.1.tar.gz" = {
13-
url = "https://github.com/apache/arrow/archive/refs/tags/apache-arrow-19.0.1.tar.gz";
14-
md5 = "8c5091da0f8fb41a47d7f4dad7b712df";
15-
sha256 = "0snqca1wlkxc22n8k2gs21g6py8r575hwwgqns3cqhc8jl28b2ac";
12+
"arrow-apache-arrow-24.0.0.tar.gz" = {
13+
url = "https://github.com/apache/arrow/archive/refs/tags/apache-arrow-24.0.0.tar.gz";
14+
md5 = "66c53bd00baa79034bd2ca167beea436";
15+
sha256 = "0a3b43avih2kiniav2rhp24awdhmxhzp4fmvlr6lqci6iwc8vqcl";
1616
};
1717
"avro-release-1.12.0.tar.gz" = {
1818
url = "https://github.com/apache/avro/archive/refs/tags/release-1.12.0.tar.gz";
@@ -349,10 +349,10 @@ let
349349
md5 = "161cbf4c347f6daadacfb749c31842f8";
350350
sha256 = "1g943v85ipgyxb62hrn54fsc5aicihrp6rwvj0pb6p19l181swym";
351351
};
352-
"xsimd-13.2.0.tar.gz" = {
353-
url = "https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.2.0.tar.gz";
354-
md5 = "f451a1c57d2a4fdc0ba663be438dced4";
355-
sha256 = "0yj4hbdfrhwkdny6fb7yq5jbw6pldp1m671jf3f5l64cahywvn7d";
352+
"xsimd-14.0.0.tar.gz" = {
353+
url = "https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz";
354+
md5 = "75c0d34cf7011924ba19978076c76dc1";
355+
sha256 = "1khd94aipz09blfxvnp53g9r4frsbx68v4yn146c2ma9jlv05php";
356356
};
357357
"xxHash-0.8.3.tar.gz" = {
358358
url = "https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.3.tar.gz";
@@ -365,9 +365,9 @@ let
365365
sha256 = "1l87r6h8qccba1yjx6n13q36yh0wrq3rzz9nnraq5ssjr2lmmipr";
366366
};
367367
"zstd-1.5.7.tar.gz" = {
368-
url = "https://github.com/facebook/zstd/archive/v1.5.7.tar.gz";
369-
md5 = "619a019adbbc4536e7fb93cdbb01af3e";
370-
sha256 = "1qrf074w0wlykrj3pgfs5qp90s374a05pa0wdvjm82djar2jimrp";
368+
url = "https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz";
369+
md5 = "780fc1896922b1bc52a4e90980cdda48";
370+
sha256 = "18vgkvh7w6zw4jn2aj1mp0yv98m4fk52ay6da0wh4pm194gyaczb";
371371
};
372372
};
373373

@@ -486,7 +486,7 @@ let
486486
"librdkafka-2.11.0.tar.gz"
487487
"pulsar-client-3.3.0.tar.gz"
488488
"flatbuffers-v1.10.0.tar.gz"
489-
"arrow-apache-arrow-19.0.1.tar.gz"
489+
"arrow-apache-arrow-24.0.0.tar.gz"
490490
"brotli-1.0.9.tar.gz"
491491
"zstd-1.5.7.tar.gz"
492492
"s2geometry-0.9.0.tar.gz"
@@ -521,7 +521,7 @@ let
521521
"simdutf-5.2.8.tar.gz"
522522
"poco-1.12.5-release.tar.gz"
523523
"icu4c-76_1-src.zip"
524-
"xsimd-13.2.0.tar.gz"
524+
"xsimd-14.0.0.tar.gz"
525525
"libxml2-2.11.7.tar.gz"
526526
"azure-storage-files-shares_12.12.0.tar.gz"
527527
"libdivide-v5.2.0.tar.gz"
@@ -557,7 +557,7 @@ let
557557
"librdkafka-2.11.0.tar.gz"
558558
"pulsar-client-3.3.0.tar.gz"
559559
"flatbuffers-v1.10.0.tar.gz"
560-
"arrow-apache-arrow-19.0.1.tar.gz"
560+
"arrow-apache-arrow-24.0.0.tar.gz"
561561
"brotli-1.0.9.tar.gz"
562562
"zstd-1.5.7.tar.gz"
563563
"s2geometry-0.9.0.tar.gz"
@@ -590,7 +590,7 @@ let
590590
"simdutf-5.2.8.tar.gz"
591591
"poco-1.12.5-release.tar.gz"
592592
"icu4c-76_1-src.zip"
593-
"xsimd-13.2.0.tar.gz"
593+
"xsimd-14.0.0.tar.gz"
594594
"libxml2-2.11.7.tar.gz"
595595
"azure-storage-files-shares_12.12.0.tar.gz"
596596
"libdivide-v5.2.0.tar.gz"
@@ -626,7 +626,7 @@ let
626626
"librdkafka-2.11.0.tar.gz"
627627
"pulsar-client-3.3.0.tar.gz"
628628
"flatbuffers-v1.10.0.tar.gz"
629-
"arrow-apache-arrow-19.0.1.tar.gz"
629+
"arrow-apache-arrow-24.0.0.tar.gz"
630630
"brotli-1.0.9.tar.gz"
631631
"zstd-1.5.7.tar.gz"
632632
"s2geometry-0.9.0.tar.gz"
@@ -659,7 +659,7 @@ let
659659
"simdutf-5.2.8.tar.gz"
660660
"poco-1.12.5-release.tar.gz"
661661
"icu4c-76_1-src.zip"
662-
"xsimd-13.2.0.tar.gz"
662+
"xsimd-14.0.0.tar.gz"
663663
"libxml2-2.11.7.tar.gz"
664664
"azure-storage-files-shares_12.12.0.tar.gz"
665665
"libdivide-v5.2.0.tar.gz"

‎thirdparty/build-thirdparty.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -964,7 +964,8 @@ build_arrow() {
964964
-DCMAKE_PREFIX_PATH=${TP_INSTALL_DIR} \
965965
-G "${CMAKE_GENERATOR}" \
966966
-DThrift_ROOT=$TP_INSTALL_DIR/ \
967-
-Dthrift_SOURCE=SYSTEM \
967+
-DARROW_THRIFT_USE_SHARED=OFF \
968+
-DThrift_SOURCE=SYSTEM \
968969
-Dxsimd_SOURCE=SYSTEM \
969970
-Dxsimd_DIR=$TP_INSTALL_DIR/share/cmake/xsimd ..
970971

‎thirdparty/vars.sh‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -229,10 +229,10 @@ PULSAR_SOURCE=pulsar-client-cpp-3.3.0
229229
PULSAR_MD5SUM="348b7e5ec39e50547668520d13a417a1"
230230

231231
# zstd
232-
ZSTD_DOWNLOAD="https://github.com/facebook/zstd/archive/v1.5.7.tar.gz"
232+
ZSTD_DOWNLOAD="https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz"
233233
ZSTD_NAME=zstd-1.5.7.tar.gz
234234
ZSTD_SOURCE=zstd-1.5.7
235-
ZSTD_MD5SUM="619a019adbbc4536e7fb93cdbb01af3e"
235+
ZSTD_MD5SUM="780fc1896922b1bc52a4e90980cdda48"
236236

237237
# brotli
238238
BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz"
@@ -247,10 +247,10 @@ FLATBUFFERS_SOURCE=flatbuffers-1.10.0
247247
FLATBUFFERS_MD5SUM="f7d19a3f021d93422b0bc287d7148cd2"
248248

249249
# arrow
250-
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-19.0.1.tar.gz"
251-
ARROW_NAME="arrow-apache-arrow-19.0.1.tar.gz"
252-
ARROW_SOURCE="arrow-apache-arrow-19.0.1"
253-
ARROW_MD5SUM="8c5091da0f8fb41a47d7f4dad7b712df"
250+
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-24.0.0.tar.gz"
251+
ARROW_NAME="arrow-apache-arrow-24.0.0.tar.gz"
252+
ARROW_SOURCE="arrow-apache-arrow-24.0.0"
253+
ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436"
254254

255255
# S2
256256
S2_DOWNLOAD="https://github.com/google/s2geometry/archive/v0.9.0.tar.gz"
@@ -447,10 +447,10 @@ ICU_SOURCE="icu"
447447
ICU_MD5SUM="f5f5c827d94af8445766c7023aca7f6b"
448448

449449
# xsimd
450-
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.2.0.tar.gz"
451-
XSIMD_NAME="xsimd-13.2.0.tar.gz"
452-
XSIMD_SOURCE="xsimd-13.2.0"
453-
XSIMD_MD5SUM="f451a1c57d2a4fdc0ba663be438dced4"
450+
XSIMD_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/14.0.0.tar.gz"
451+
XSIMD_NAME="xsimd-14.0.0.tar.gz"
452+
XSIMD_SOURCE="xsimd-14.0.0"
453+
XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1"
454454

455455
# libxml2
456456
LIBXML2_DOWNLOAD="https://github.com/GNOME/libxml2/archive/refs/tags/v2.11.7.tar.gz"

0 commit comments

Comments
 (0)