Merge branch 'PushGeneratorWithBackpressure' into combined3

Author: Rafał Hibner

.env

@@ -92,7 +92,7 @@ TZ=UTC
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
 # docker tags more readable.
-VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625"    # 2024.06.15 Release
+VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1"    # 2025.09.17 Release
 
 # This must be updated when we update
 # ci/docker/python-*-windows-*.dockerfile or the vcpkg config.

ci/conda_env_cpp.txt

@@ -31,16 +31,16 @@ gflags
 glog
 gmock>=1.10.0
 google-cloud-cpp>=1.34.0
-grpc-cpp<=1.50.1
 gtest>=1.10.0
+libgrpc
 libprotobuf
 libutf8proc
 lz4-c
 make
 meson
 ninja
 nodejs
-orc
+orc<2.1.0
 pkg-config
 python
 rapidjson

ci/docker/python-wheel-windows-test-vs2022-base.dockerfile

@@ -51,7 +51,7 @@ SHELL ["cmd", "/S", "/C"]
 
 # Install git, wget, minio
 RUN choco install --no-progress -r -y git wget
-RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z `
+RUN curl https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2025-01-20T14-49-07Z `
     --output "C:\Windows\Minio.exe"
 
 # Install the GCS testbench using a well-known Python version.

ci/scripts/python_wheel_windows_build.bat

@@ -60,7 +60,7 @@ set CMAKE_GENERATOR=Visual Studio 17 2022
 set CMAKE_PLATFORM=x64
 set VCPKG_ROOT=C:\vcpkg
 set VCPKG_FEATURE_FLAGS=-manifests
-set VCGPK_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
+set VCPKG_TARGET_TRIPLET=amd64-windows-static-md-%CMAKE_BUILD_TYPE%
 
 mkdir C:\arrow-build
 pushd C:\arrow-build
@@ -100,7 +100,7 @@ cmake ^
     -DMSVC_LINK_VERBOSE=ON ^
     -DPARQUET_REQUIRE_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% ^
     -DVCPKG_MANIFEST_MODE=OFF ^
-    -DVCPKG_TARGET_TRIPLET=%VCGPK_TARGET_TRIPLET% ^
+    -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^
     -Dxsimd_SOURCE=BUNDLED ^
     -G "%CMAKE_GENERATOR%" ^
     -A "%CMAKE_PLATFORM%" ^

ci/vcpkg/ports.patch

@@ -1,25 +1,26 @@
 diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
-index 7cab6f726..697ab1bb4 100644
+index 6788bc7b7f..6b689dedf0 100644
 --- a/ports/curl/portfile.cmake
 +++ b/ports/curl/portfile.cmake
-@@ -84,9 +84,12 @@ vcpkg_cmake_configure(
-         -DBUILD_TESTING=OFF
+@@ -83,10 +83,13 @@ vcpkg_cmake_configure(
          -DENABLE_CURL_MANUAL=OFF
-         -DCURL_CA_FALLBACK=ON
+         -DIMPORT_LIB_SUFFIX=   # empty
+         -DSHARE_LIB_OBJECT=OFF
 +        -DCURL_CA_PATH=none
 +        -DCURL_CA_BUNDLE=none
+         -DCURL_USE_PKGCONFIG=ON
          -DCMAKE_DISABLE_FIND_PACKAGE_Perl=ON
-     OPTIONS_DEBUG
-         -DENABLE_DEBUG=ON
+     MAYBE_UNUSED_VARIABLES
+         PKG_CONFIG_EXECUTABLE
 +        ${EXTRA_ARGS_DEBUG}
  )
  vcpkg_cmake_install()
  vcpkg_copy_pdbs()
 diff --git a/ports/llvm/portfile.cmake b/ports/llvm/portfile.cmake
-index a79c72a59..6b7fa6a66 100644
+index 7764357a6d..da7374ecec 100644
 --- a/ports/llvm/portfile.cmake
 +++ b/ports/llvm/portfile.cmake
-@@ -292,6 +292,8 @@ vcpkg_cmake_configure(
+@@ -302,6 +302,8 @@ vcpkg_cmake_configure(
          ${FEATURE_OPTIONS}
      MAYBE_UNUSED_VARIABLES 
          COMPILER_RT_ENABLE_IOS
@@ -28,92 +29,52 @@ index a79c72a59..6b7fa6a66 100644
  )
 
  vcpkg_cmake_install(ADD_BIN_TO_PATH)
-diff --git a/ports/snappy/portfile.cmake b/ports/snappy/portfile.cmake
-index 0312b2ae1..fdb576b5f 100644
---- a/ports/snappy/portfile.cmake
-+++ b/ports/snappy/portfile.cmake
-@@ -8,5 +8,6 @@ vcpkg_from_github(
-         fix_clang-cl_build.patch
-         no-werror.patch
-         pkgconfig.diff
-+        "snappy-disable-bmi.patch"
- )
- file(COPY "${CURRENT_PORT_DIR}/snappy.pc.in" DESTINATION "${SOURCE_PATH}")
-diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
+diff --git a/ports/orc/orc-fix-exception-propagation.diff b/ports/orc/orc-fix-exception-propagation.diff
 new file mode 100644
-index 000000000..e839c93a4
+index 0000000000..25568e70cd
 --- /dev/null
-+++ b/ports/snappy/snappy-disable-bmi.patch
-@@ -0,0 +1,19 @@
-+diff --git a/snappy.cc b/snappy.cc
-+index d414718..7b49d2a 100644
-+--- a/snappy.cc
-++++ b/snappy.cc
-+@@ -1014,14 +1014,10 @@ static inline void Report(const char *algorithm, size_t compressed_size,
-+ static inline uint32_t ExtractLowBytes(const uint32_t& v, int n) {
-+   assert(n >= 0);
-+   assert(n <= 4);
-+-#if SNAPPY_HAVE_BMI2
-+-  return _bzhi_u32(v, 8 * n);
-+-#else
-+   // This needs to be wider than uint32_t otherwise `mask << 32` will be
-+   // undefined.
-+   uint64_t mask = 0xffffffff;
-+   return v & ~(mask << (8 * n));
-+-#endif
-+ }
++++ b/ports/orc/orc-fix-exception-propagation.diff
+@@ -0,0 +1,30 @@
++diff --git a/c++/src/Timezone.cc b/c++/src/Timezone.cc
++index 384f8ea99..07c75e0a7 100644
++--- a/c++/src/Timezone.cc
+++++ b/c++/src/Timezone.cc
++@@ -696,12 +696,21 @@ namespace orc {
++     std::string filename_;
++     mutable std::unique_ptr<TimezoneImpl> impl_;
++     mutable std::once_flag initialized_;
+++    mutable std::exception_ptr init_exception_;
++ 
++     TimezoneImpl* getImpl() const {
++-      std::call_once(initialized_, [&]() {
++-        auto buffer = loadTZDB(filename_);
++-        impl_ = std::make_unique<TimezoneImpl>(filename_, std::move(buffer));
++-      });
+++        std::call_once(initialized_, [&]() {
+++          try {
+++            auto buffer = loadTZDB(filename_);
+++            impl_ = std::make_unique<TimezoneImpl>(filename_, std::move(buffer));
+++          } catch (...) {
+++            // If initialization failed, re-throw the exception
+++            init_exception_ = std::current_exception();
+++          }
+++        });
+++      if (init_exception_) {
+++        std::rethrow_exception(init_exception_);
+++      }
++       return impl_.get();
++     }
 + 
-+ static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {
-diff --git a/ports/thrift/portfile.cmake b/ports/thrift/portfile.cmake
-index 1501782..71d2147 100644
---- a/ports/thrift/portfile.cmake
-+++ b/ports/thrift/portfile.cmake
-@@ -12,7 +12,7 @@ vcpkg_find_acquire_program(BISON)
- vcpkg_from_github(
-     OUT_SOURCE_PATH SOURCE_PATH
-     REPO apache/thrift
--    REF "${VERSION}"
-+    REF "v${VERSION}"
-     SHA512 5e4ee9870b30fe5ba484d39781c435716f7f3903793dc8aae96594ca813b1a5a73363b84719038ca8fa3ab8ef0a419a28410d936ff7b3bbadf36fc085a6883ae
-     HEAD_REF master
-     PATCHES
-diff --git a/ports/thrift/vcpkg.json b/ports/thrift/vcpkg.json
-index 2d5a854..9ff49ec 100644
---- a/ports/thrift/vcpkg.json
-+++ b/ports/thrift/vcpkg.json
-@@ -1,6 +1,7 @@
- {
-   "name": "thrift",
-   "version": "0.20.0",
-+  "port-version": 1,
-   "description": "Apache Thrift is a software project spanning a variety of programming languages and use cases. Our goal is to make reliable, performant communication and data serialization across languages as efficient and seamless as possible.",
-   "homepage": "https://github.com/apache/thrift",
-   "license": "Apache-2.0",
-diff --git a/versions/baseline.json b/versions/baseline.json
-index c6ce736..9ad1d63 100644
---- a/versions/baseline.json
-+++ b/versions/baseline.json
-@@ -8622,7 +8622,7 @@
-     },
-     "thrift": {
-       "baseline": "0.20.0",
--      "port-version": 0
-+      "port-version": 1
-     },
-     "tidy-html5": {
-       "baseline": "5.8.0",
-diff --git a/versions/t-/thrift.json b/versions/t-/thrift.json
-index 3db38c5..7464bde 100644
---- a/versions/t-/thrift.json
-+++ b/versions/t-/thrift.json
-@@ -1,5 +1,10 @@
- {
-   "versions": [
-+    {
-+      "git-tree": "13757a6b05741cf3c9c39e3a1dcc5e5cd685e025",
-+      "version": "0.20.0",
-+      "port-version": 1
-+    },
-     {
-       "git-tree": "6855be1ce96497811d4eb0a9879baf6cf1b3610c",
-       "version": "0.20.0",
+diff --git a/ports/orc/portfile.cmake b/ports/orc/portfile.cmake
+index 77ebf41ec3..4d065594a7 100644
+--- a/ports/orc/portfile.cmake
++++ b/ports/orc/portfile.cmake
+@@ -6,6 +6,8 @@ vcpkg_from_github(
+   REF "v${VERSION}"
+   SHA512 eabee16a6e984452a8cb715d0524041b20dd1bd88d78bb32534db93e5dbdd786aa4df8c05975406cb0728241eb3025a506c4fefb8c334ef0d8a27e6cb920d44c
+   HEAD_REF master
++  PATCHES
++    orc-fix-exception-propagation.diff
+ )
+ 
+ file(REMOVE "${SOURCE_PATH}/cmake_modules/FindGTest.cmake")

cpp/src/arrow/flight/CMakeLists.txt

@@ -189,6 +189,19 @@ set(ARROW_FLIGHT_SRCS
     transport/grpc/util_internal.cc
     types.cc)
 
+# Handle Unity build header conflicts on Windows.
+if(CMAKE_UNITY_BUILD AND WIN32)
+  set_source_files_properties(client.cc
+                              cookie_internal.cc
+                              serialization_internal.cc
+                              server.cc
+                              transport/grpc/serialization_internal.cc
+                              transport/grpc/protocol_grpc_internal.cc
+                              transport/grpc/util_internal.cc
+                              types.cc
+                              PROPERTIES SKIP_UNITY_BUILD_INCLUSION TRUE)
+endif()
+
 if(ARROW_WITH_OPENTELEMETRY)
   list(APPEND ARROW_FLIGHT_SRCS otel_logging.cc)
 endif()

cpp/src/arrow/flight/sql/odbc/flight_sql/CMakeLists.txt

@@ -99,10 +99,12 @@ if(WIN32)
                          system_dsn.cc)
 endif()
 
-target_link_libraries(arrow_odbc_spi_impl PUBLIC odbcabstraction arrow_flight_sql_shared)
+target_link_libraries(arrow_odbc_spi_impl PUBLIC odbcabstraction arrow_flight_sql_shared
+                                                 arrow_compute_shared Boost::locale)
 
-if(MSVC)
-  target_link_libraries(arrow_odbc_spi_impl PUBLIC Boost::locale)
+# Link libraries on MINGW64 and macOS
+if(MINGW OR APPLE)
+  target_link_libraries(arrow_odbc_spi_impl PUBLIC ${ODBCINST})
 endif()
 
 set_target_properties(arrow_odbc_spi_impl
@@ -121,7 +123,7 @@ set_target_properties(arrow_odbc_spi_impl_cli
 target_link_libraries(arrow_odbc_spi_impl_cli arrow_odbc_spi_impl)
 
 # Unit tests
-add_arrow_test(arrow_odbc_spi_impl_test
+add_arrow_test(odbc_spi_impl_test
                SOURCES
                accessors/boolean_array_accessor_test.cc
                accessors/binary_array_accessor_test.cc

cpp/src/arrow/flight/sql/odbc/flight_sql/accessors/binary_array_accessor.cc

@@ -56,8 +56,8 @@ inline RowStatus MoveSingleCellToBinaryBuffer(ColumnBinding* binding, BinaryArra
     value_offset = -1;
   }
 
-  if (binding->strlen_buffer) {
-    binding->strlen_buffer[i] = static_cast<ssize_t>(remaining_length);
+  if (binding->str_len_buffer) {
+    binding->str_len_buffer[i] = static_cast<ssize_t>(remaining_length);
   }
 
   return result;
@@ -72,15 +72,15 @@ BinaryArrayFlightSqlAccessor<TARGET_TYPE>::BinaryArrayFlightSqlAccessor(Array* a
 
 template <>
 RowStatus
-BinaryArrayFlightSqlAccessor<odbcabstraction::CDataType_BINARY>::MoveSingleCell_impl(
+BinaryArrayFlightSqlAccessor<odbcabstraction::CDataType_BINARY>::MoveSingleCellImpl(
     ColumnBinding* binding, int64_t arrow_row, int64_t i, int64_t& value_offset,
     bool update_value_offset, odbcabstraction::Diagnostics& diagnostics) {
   return MoveSingleCellToBinaryBuffer(binding, this->GetArray(), arrow_row, i,
                                       value_offset, update_value_offset, diagnostics);
 }
 
 template <CDataType TARGET_TYPE>
-size_t BinaryArrayFlightSqlAccessor<TARGET_TYPE>::GetCellLength_impl(
+size_t BinaryArrayFlightSqlAccessor<TARGET_TYPE>::GetCellLengthImpl(
     ColumnBinding* binding) const {
   return binding->buffer_length;
 }

cpp/src/arrow/flight/sql/odbc/flight_sql/accessors/binary_array_accessor.h

@@ -34,11 +34,11 @@ class BinaryArrayFlightSqlAccessor
  public:
   explicit BinaryArrayFlightSqlAccessor(Array* array);
 
-  RowStatus MoveSingleCell_impl(ColumnBinding* binding, int64_t arrow_row, int64_t i,
-                                int64_t& value_offset, bool update_value_offset,
-                                odbcabstraction::Diagnostics& diagnostics);
+  RowStatus MoveSingleCellImpl(ColumnBinding* binding, int64_t arrow_row, int64_t i,
+                               int64_t& value_offset, bool update_value_offset,
+                               odbcabstraction::Diagnostics& diagnostics);
 
-  size_t GetCellLength_impl(ColumnBinding* binding) const;
+  size_t GetCellLengthImpl(ColumnBinding* binding) const;
 };
 
 }  // namespace flight_sql

cpp/src/arrow/flight/sql/odbc/flight_sql/accessors/binary_array_accessor_test.cc

@@ -35,12 +35,12 @@ TEST(BinaryArrayAccessor, Test_CDataType_BINARY_Basic) {
 
   BinaryArrayFlightSqlAccessor<odbcabstraction::CDataType_BINARY> accessor(array.get());
 
-  size_t max_strlen = 64;
-  std::vector<char> buffer(values.size() * max_strlen);
-  std::vector<ssize_t> strlen_buffer(values.size());
+  size_t max_str_len = 64;
+  std::vector<char> buffer(values.size() * max_str_len);
+  std::vector<ssize_t> str_len_buffer(values.size());
 
   ColumnBinding binding(odbcabstraction::CDataType_BINARY, 0, 0, buffer.data(),
-                        max_strlen, strlen_buffer.data());
+                        max_str_len, str_len_buffer.data());
 
   int64_t value_offset = 0;
   odbcabstraction::Diagnostics diagnostics("Foo", "Foo", OdbcVersion::V_3);
@@ -49,12 +49,13 @@ TEST(BinaryArrayAccessor, Test_CDataType_BINARY_Basic) {
                                      diagnostics, nullptr));
 
   for (int i = 0; i < values.size(); ++i) {
-    ASSERT_EQ(values[i].length(), strlen_buffer[i]);
+    ASSERT_EQ(values[i].length(), str_len_buffer[i]);
     // Beware that CDataType_BINARY values are not null terminated.
     // It's safe to create a std::string from this data because we know it's
     // ASCII, this doesn't work with arbitrary binary data.
-    ASSERT_EQ(values[i], std::string(buffer.data() + i * max_strlen,
-                                     buffer.data() + i * max_strlen + strlen_buffer[i]));
+    ASSERT_EQ(values[i],
+              std::string(buffer.data() + i * max_str_len,
+                          buffer.data() + i * max_str_len + str_len_buffer[i]));
   }
 }
 
@@ -65,12 +66,12 @@ TEST(BinaryArrayAccessor, Test_CDataType_BINARY_Truncation) {
 
   BinaryArrayFlightSqlAccessor<odbcabstraction::CDataType_BINARY> accessor(array.get());
 
-  size_t max_strlen = 8;
-  std::vector<char> buffer(values.size() * max_strlen);
-  std::vector<ssize_t> strlen_buffer(values.size());
+  size_t max_str_len = 8;
+  std::vector<char> buffer(values.size() * max_str_len);
+  std::vector<ssize_t> str_len_buffer(values.size());
 
   ColumnBinding binding(odbcabstraction::CDataType_BINARY, 0, 0, buffer.data(),
-                        max_strlen, strlen_buffer.data());
+                        max_str_len, str_len_buffer.data());
 
   std::stringstream ss;
   int64_t value_offset = 0;
@@ -83,13 +84,13 @@ TEST(BinaryArrayAccessor, Test_CDataType_BINARY_Truncation) {
     int64_t original_value_offset = value_offset;
     ASSERT_EQ(1, accessor.GetColumnarData(&binding, 0, 1, value_offset, true, diagnostics,
                                           nullptr));
-    ASSERT_EQ(values[0].length() - original_value_offset, strlen_buffer[0]);
+    ASSERT_EQ(values[0].length() - original_value_offset, str_len_buffer[0]);
 
     int64_t chunk_length = 0;
     if (value_offset == -1) {
-      chunk_length = strlen_buffer[0];
+      chunk_length = str_len_buffer[0];
     } else {
-      chunk_length = max_strlen;
+      chunk_length = max_str_len;
     }
 
     // Beware that CDataType_BINARY values are not null terminated.