Merge remote-tracking branch 'upstream/main' into apachegh-43683-pandas-string-dtype

Author: jorisvandenbossche

ci/scripts/python_wheel_windows_build.bat

@@ -20,6 +20,12 @@
 echo "Building windows wheel..."
 
 call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
+@echo on
+
+@REM Install a more recent msvcp140.dll in C:\Windows\System32
+choco install -r -y --no-progress vcredist140
+choco upgrade -r -y --no-progress vcredist140
+dir C:\Windows\System32\msvcp140.dll
 
 echo "=== (%PYTHON_VERSION%) Clear output directories and leftovers ==="
 del /s /q C:\arrow-build
@@ -121,7 +127,27 @@ set ARROW_HOME=C:\arrow-dist
 set CMAKE_PREFIX_PATH=C:\arrow-dist
 
 pushd C:\arrow\python
-@REM bundle the msvc runtime
-cp "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Redist\MSVC\14.28.29325\x64\Microsoft.VC142.CRT\msvcp140.dll" pyarrow\
+
+@REM Bundle the C++ runtime
+cp C:\Windows\System32\msvcp140.dll pyarrow\
+
+@REM Build wheel
 python setup.py bdist_wheel || exit /B 1
+
+@REM Repair the wheel with delvewheel
+@REM
+@REM Since we bundled the Arrow C++ libraries ourselves, we only need to
+@REM mangle msvcp140.dll so as to avoid ABI issues when msvcp140.dll is
+@REM required by multiple Python libraries in the same process.
+@REM
+@REM For now this requires a custom version of delvewheel:
+@REM https://github.com/adang1345/delvewheel/pull/59
+pip install https://github.com/pitrou/delvewheel/archive/refs/heads/fixes-for-arrow.zip || exit /B 1
+
+for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1
+echo "Wheel name: %WHEEL_NAME%"
+
+delvewheel repair -vv --mangle-only=msvcp140.dll --no-patch ^
+    -w repaired_wheels %WHEEL_NAME% || exit /B 1
+
 popd

ci/scripts/python_wheel_windows_test.bat

@@ -48,7 +48,7 @@ set PYTHON_CMD=py -%PYTHON%
 %PYTHON_CMD% -m pip install -r C:\arrow\python\requirements-wheel-test.txt || exit /B 1
 
 @REM Install the built wheels
-%PYTHON_CMD% -m pip install --no-index --find-links=C:\arrow\python\dist\ pyarrow || exit /B 1
+%PYTHON_CMD% -m pip install --no-index --find-links=C:\arrow\python\repaired_wheels pyarrow || exit /B 1
 
 @REM Test that the modules are importable
 %PYTHON_CMD% -c "import pyarrow" || exit /B 1
@@ -65,7 +65,7 @@ set PYTHON_CMD=py -%PYTHON%
 %PYTHON_CMD% -c "import pyarrow.substrait" || exit /B 1
 
 @REM Validate wheel contents
-%PYTHON_CMD% C:\arrow\ci\scripts\python_wheel_validate_contents.py --path C:\arrow\python\dist || exit /B 1
+%PYTHON_CMD% C:\arrow\ci\scripts\python_wheel_validate_contents.py --path C:\arrow\python\repaired_wheels || exit /B 1
 
 @rem Download IANA Timezone Database for ORC C++
 curl https://cygwin.osuosl.org/noarch/release/tzdata/tzdata-2024a-1.tar.xz --output tzdata.tar.xz || exit /B

cpp/src/parquet/arrow/arrow_schema_test.cc

@@ -832,6 +832,27 @@ TEST_F(TestConvertParquetSchema, IllegalParquetNestedSchema) {
         Invalid, testing::HasSubstr("LIST-annotated groups must not be repeated."),
         ConvertSchema(parquet_fields));
   }
+  // List<List<>>: outer list is two-level encoding, inner list is empty.
+  //
+  // optional group my_list (LIST) {
+  //   repeated group array (LIST) {
+  //     repeated group list {
+  //     }
+  //   }
+  // }
+  {
+    auto list = GroupNode::Make("list", Repetition::REPEATED, {});
+    auto array =
+        GroupNode::Make("array", Repetition::REPEATED, {list}, ConvertedType::LIST);
+    std::vector<NodePtr> parquet_fields;
+    parquet_fields.push_back(
+        GroupNode::Make("my_list", Repetition::OPTIONAL, {array}, ConvertedType::LIST));
+
+    EXPECT_RAISES_WITH_MESSAGE_THAT(
+        Invalid,
+        testing::HasSubstr("LIST-annotated groups must have at least one child."),
+        ConvertSchema(parquet_fields));
+  }
 }
 
 Status ArrowSchemaToParquetMetadata(std::shared_ptr<::arrow::Schema>& arrow_schema,

cpp/src/parquet/arrow/schema.cc

@@ -676,9 +676,14 @@ Status ListToSchemaField(const GroupNode& group, LevelInfo current_levels,
           return Status::Invalid("Group with one repeated child must be LIST-annotated.");
         }
         // LIST-annotated group with three-level encoding cannot be repeated.
-        if (repeated_field->is_group() &&
-            !static_cast<const GroupNode&>(*repeated_field).field(0)->is_repeated()) {
-          return Status::Invalid("LIST-annotated groups must not be repeated.");
+        if (repeated_field->is_group()) {
+          auto& repeated_group_field = static_cast<const GroupNode&>(*repeated_field);
+          if (repeated_group_field.field_count() == 0) {
+            return Status::Invalid("LIST-annotated groups must have at least one child.");
+          }
+          if (!repeated_group_field.field(0)->is_repeated()) {
+            return Status::Invalid("LIST-annotated groups must not be repeated.");
+          }
         }
         RETURN_NOT_OK(
             NodeToSchemaField(*repeated_field, current_levels, ctx, out, child_field));

dev/tasks/python-wheels/github.windows.yml

@@ -63,17 +63,17 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: wheel
-          path: arrow/python/dist/*.whl
+          path: arrow/python/repaired_wheels/*.whl
 
       - name: Test wheel
         shell: cmd
         run: |
           cd arrow
           archery docker run python-wheel-windows-test
 
-      {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }}
-      {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }}
-      {{ macros.github_upload_wheel_scientific_python("arrow/python/dist/*.whl")|indent }}
+      {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
+      {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
+      {{ macros.github_upload_wheel_scientific_python("arrow/repaired_wheels/repaired_wheels/*.whl")|indent }}
 
       {% if arrow.is_default_branch() %}
       - name: Push Docker Image