chore(lambda-layer): adapt build for pyarrow 24 (scikit-build-core, drop py3.9)

PyArrow 24 dropped setup.py and moved to pyproject.toml + scikit-build-core.
The legacy `setup.py build_ext --bundle-arrow-cpp bdist_wheel` invocation
fails because arrow/python/setup.py no longer exists.

- Use `python -m build --wheel --no-isolation` and export
  PYARROW_BUNDLE_ARROW_CPP=ON (replaces the --bundle-arrow-cpp flag).
- Drop the PYARROW_WITH_{PLASMA,CSV,JSON,COMPUTE,FILESYSTEM} and
  ARROW_PRE_0_15_IPC_FORMAT exports — pyarrow 24's CMakeLists no longer
  defines those options.
- Install build deps (build, scikit-build-core, libcst>=1.8.6) and pin
  cython>=3.1 in both Dockerfiles.
- Drop the Python 3.9 layer build: pyarrow 24 requires Python >= 3.10.

Author: kukushking

building/build-lambda-layers.sh

@@ -21,17 +21,6 @@ pushd lambda
 # Building all related docker images
 ./build-docker-images.sh $PYTHON_VERSION
 
-# Python 3.9
-if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.9" ]]
-then
-  docker run \
-    --volume "$DIR_NAME":/aws-sdk-pandas/ \
-    --workdir /aws-sdk-pandas/building/lambda \
-    --rm \
-    awswrangler-build-py39 \
-    build-lambda-layer.sh "${VERSION}-py3.9${ARCH_SUFFIX}" "ninja-build"
-fi
-
 # Python 3.10
 if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.10" ]]
 then

building/lambda/Dockerfile

@@ -35,14 +35,15 @@ RUN ln -s /usr/bin/gcc10-ld /usr/bin/ld
 FROM ${python_version}
 COPY pyproject.toml uv.lock ./
 
-# Setuptools is a build dependency of arrow and runtime dependency of some of our dependencies (mainly redshift-connector).
-# Remove when arrow version shipped with lambda layers and dependencies are updated.
+# Setuptools is a runtime dependency of some of our dependencies (mainly redshift-connector).
 RUN pip3 install --upgrade pip wheel setuptools>=78.1.1 setuptools_scm>=8
 RUN pip3 install --upgrade urllib3==1.26.16  # temporary to avoid https://github.com/urllib3/urllib3/issues/2168 (TODO remove when the AL2 image updates to support OpenSSL 1.1.1+)
-# In new CMake 4, compatibility with CMake < 3.5 has been removed. 
+# In new CMake 4, compatibility with CMake < 3.5 has been removed.
 # Unpin CMake when arrow version shipped with lambda layers is updated.
 RUN pip3 install --upgrade cmake==3.31.6
-RUN pip3 install --upgrade six cython hypothesis uv
+# pyarrow 24+ build deps: scikit-build-core replaces setuptools; cython>=3.1 and
+# libcst (used for build-time stub docstring extraction) are required.
+RUN pip3 install --upgrade six "cython>=3.1" hypothesis uv build scikit-build-core "libcst>=1.8.6"
 
 ENV PIP_NO_BINARY="numpy,pandas"
 ENV UV_PROJECT_ENVIRONMENT="/var/lang"

building/lambda/Dockerfile.al2023

@@ -24,13 +24,14 @@ WORKDIR /root
 FROM ${python_version}
 COPY pyproject.toml uv.lock ./
 
-# Setuptools is a build dependency of arrow and runtime dependency of some of our dependencies (mainly redshift-connector).
-# Remove when arrow version shipped with lambda layers and dependencies are updated.
+# Setuptools is a runtime dependency of some of our dependencies (mainly redshift-connector).
 RUN pip3 install --upgrade pip wheel setuptools>=78.1.1 setuptools_scm>=8
-# In new CMake 4, compatibility with CMake < 3.5 has been removed. 
+# In new CMake 4, compatibility with CMake < 3.5 has been removed.
 # Unpin CMake when arrow version shipped with lambda layers is updated.
 RUN pip3 install --upgrade cmake==3.31.6
-RUN pip3 install --upgrade six cython hypothesis uv
+# pyarrow 24+ build deps: scikit-build-core replaces setuptools; cython>=3.1 and
+# libcst (used for build-time stub docstring extraction) are required.
+RUN pip3 install --upgrade six "cython>=3.1" hypothesis uv build scikit-build-core "libcst>=1.8.6"
 
 ENV PIP_NO_BINARY="numpy,pandas"
 ENV UV_PROJECT_ENVIRONMENT="/var/lang"

building/lambda/build-docker-images.sh

@@ -8,16 +8,6 @@ export DOCKER_BUILDKIT=1
 
 PYTHON_VERSION=${1:-ALL}
 
-# Python 3.9
-if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.9" ]]
-then
-  docker build \
-    --pull \
-    --tag awswrangler-build-py39 \
-    --build-arg base_image=public.ecr.aws/lambda/python:3.9 \
-    .
-fi
-
 # Python 3.10
 if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.10" ]]
 then

building/lambda/build-lambda-layer.sh

@@ -59,24 +59,20 @@ popd
 pushd arrow/python
 
 export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH}}
-export ARROW_PRE_0_15_IPC_FORMAT=0
+export PYARROW_BUNDLE_ARROW_CPP=ON
 export PYARROW_WITH_HDFS=0
 export PYARROW_WITH_FLIGHT=0
 export PYARROW_WITH_GANDIVA=0
 export PYARROW_WITH_ORC=0
 export PYARROW_WITH_CUDA=0
-export PYARROW_WITH_PLASMA=0
 export PYARROW_WITH_PARQUET=1
 export PYARROW_WITH_DATASET=1
-export PYARROW_WITH_FILESYSTEM=1
-export PYARROW_WITH_CSV=1
-export PYARROW_WITH_JSON=1
-export PYARROW_WITH_COMPUTE=1
-
-python3 setup.py build_ext \
-  --build-type=release \
-  --bundle-arrow-cpp \
-  bdist_wheel
+
+# pyarrow 24+ dropped setup.py in favour of scikit-build-core via pyproject.toml.
+# --no-isolation reuses build deps already installed in the image (scikit-build-core,
+# cython>=3.1, libcst, setuptools_scm, numpy) instead of pip resolving them again.
+python3 -m build --wheel . --no-isolation \
+  -C cmake.build-type=Release
 
 pip3 install dist/pyarrow-*.whl -t /aws-sdk-pandas/dist/pyarrow_files