Skip to content

chore(lambda-layer): bump pyarrow to 24.0.0 #3375

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kukushking merged 7 commits into from
Jun 24, 2026
Merged

chore(lambda-layer): bump pyarrow to 24.0.0 #3375

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2cddaa1
chore(lambda-layer): bump pyarrow to 24.0.0
PatrickBunker Jun 17, 2026
2403db2
Merge branch 'main' into main
PatrickBunker Jun 18, 2026
b8d684f
Merge branch 'main' into main
kukushking Jun 22, 2026
937aa34
Merge branch 'main' into main
kukushking Jun 22, 2026
0dcb4da
chore(lambda-layer): adapt build for pyarrow 24 (scikit-build-core, d…
kukushking Jun 23, 2026
0822432
chore(lambda-layer): install rustup in AL2 image for libcst source build
kukushking Jun 23, 2026
a34f85f
chore(lambda-layer): symlink cc to gcc10 for libcst Rust build
kukushking Jun 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions building/build-lambda-layers.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,6 @@ pushd lambda
# Building all related docker images
./build-docker-images.sh $PYTHON_VERSION

# Python 3.9
if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.9" ]]
then
docker run \
--volume "$DIR_NAME":/aws-sdk-pandas/ \
--workdir /aws-sdk-pandas/building/lambda \
--rm \
awswrangler-build-py39 \
build-lambda-layer.sh "${VERSION}-py3.9${ARCH_SUFFIX}" "ninja-build"
fi

# Python 3.10
if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.10" ]]
then
Expand Down
17 changes: 13 additions & 4 deletions building/lambda/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ ENV CXX=/usr/bin/gcc10-c++
ENV LD=/usr/bin/gcc10-gcc

RUN ln -s /usr/bin/gcc10-gcc /usr/bin/gcc
RUN ln -s /usr/bin/gcc10-gcc /usr/bin/cc
RUN ln -s /usr/bin/gcc10-g++ /usr/bin/g++
RUN ln -s /usr/bin/gcc10-nm /usr/bin/nm
RUN ln -s /usr/bin/gcc10-ar /usr/bin/ar
Expand All @@ -35,14 +36,22 @@ RUN ln -s /usr/bin/gcc10-ld /usr/bin/ld
FROM ${python_version}
COPY pyproject.toml uv.lock ./

# Setuptools is a build dependency of arrow and runtime dependency of some of our dependencies (mainly redshift-connector).
# Remove when arrow version shipped with lambda layers and dependencies are updated.
# Setuptools is a runtime dependency of some of our dependencies (mainly redshift-connector).
RUN pip3 install --upgrade pip wheel setuptools>=78.1.1 setuptools_scm>=8
RUN pip3 install --upgrade urllib3==1.26.16 # temporary to avoid https://github.com/urllib3/urllib3/issues/2168 (TODO remove when the AL2 image updates to support OpenSSL 1.1.1+)
# In new CMake 4, compatibility with CMake < 3.5 has been removed.
# In new CMake 4, compatibility with CMake < 3.5 has been removed.
# Unpin CMake when arrow version shipped with lambda layers is updated.
RUN pip3 install --upgrade cmake==3.31.6
RUN pip3 install --upgrade six cython hypothesis uv
# libcst (a pyarrow 24+ build dep, used for build-time stub docstring extraction)
# only ships manylinux_2_28 aarch64 wheels and falls back to a Rust build on AL2's
# glibc 2.26. Install rustup so cargo is on PATH for that source build.
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
sh -s -- -y --profile minimal --default-toolchain stable
ENV PATH="/root/.cargo/bin:${PATH}"

# pyarrow 24+ build deps: scikit-build-core replaces setuptools; cython>=3.1 and
# libcst (used for build-time stub docstring extraction) are required.
RUN pip3 install --upgrade six "cython>=3.1" hypothesis uv build scikit-build-core "libcst>=1.8.6"

ENV PIP_NO_BINARY="numpy,pandas"
ENV UV_PROJECT_ENVIRONMENT="/var/lang"
Expand Down
9 changes: 5 additions & 4 deletions building/lambda/Dockerfile.al2023
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,14 @@ WORKDIR /root
FROM ${python_version}
COPY pyproject.toml uv.lock ./

# Setuptools is a build dependency of arrow and runtime dependency of some of our dependencies (mainly redshift-connector).
# Remove when arrow version shipped with lambda layers and dependencies are updated.
# Setuptools is a runtime dependency of some of our dependencies (mainly redshift-connector).
RUN pip3 install --upgrade pip wheel setuptools>=78.1.1 setuptools_scm>=8
# In new CMake 4, compatibility with CMake < 3.5 has been removed.
# In new CMake 4, compatibility with CMake < 3.5 has been removed.
# Unpin CMake when arrow version shipped with lambda layers is updated.
RUN pip3 install --upgrade cmake==3.31.6
RUN pip3 install --upgrade six cython hypothesis uv
# pyarrow 24+ build deps: scikit-build-core replaces setuptools; cython>=3.1 and
# libcst (used for build-time stub docstring extraction) are required.
RUN pip3 install --upgrade six "cython>=3.1" hypothesis uv build scikit-build-core "libcst>=1.8.6"

ENV PIP_NO_BINARY="numpy,pandas"
ENV UV_PROJECT_ENVIRONMENT="/var/lang"
Expand Down
10 changes: 0 additions & 10 deletions building/lambda/build-docker-images.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,6 @@ export DOCKER_BUILDKIT=1

PYTHON_VERSION=${1:-ALL}

# Python 3.9
if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.9" ]]
then
docker build \
--pull \
--tag awswrangler-build-py39 \
--build-arg base_image=public.ecr.aws/lambda/python:3.9 \
.
fi

# Python 3.10
if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.10" ]]
then
Expand Down
20 changes: 8 additions & 12 deletions building/lambda/build-lambda-layer.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ popd
rm -rf dist arrow

export ARROW_HOME=$(pwd)/dist
export ARROW_VERSION=22.0.0
export ARROW_VERSION=24.0.0
export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH
export CMAKE_PREFIX_PATH=$ARROW_HOME:$CMAKE_PREFIX_PATH
export SETUPTOOLS_SCM_PRETEND_VERSION=$ARROW_VERSION
Expand Down Expand Up @@ -59,24 +59,20 @@ popd
pushd arrow/python

export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH}}
export ARROW_PRE_0_15_IPC_FORMAT=0
export PYARROW_BUNDLE_ARROW_CPP=ON
export PYARROW_WITH_HDFS=0
export PYARROW_WITH_FLIGHT=0
export PYARROW_WITH_GANDIVA=0
export PYARROW_WITH_ORC=0
export PYARROW_WITH_CUDA=0
export PYARROW_WITH_PLASMA=0
export PYARROW_WITH_PARQUET=1
export PYARROW_WITH_DATASET=1
export PYARROW_WITH_FILESYSTEM=1
export PYARROW_WITH_CSV=1
export PYARROW_WITH_JSON=1
export PYARROW_WITH_COMPUTE=1

python3 setup.py build_ext \
--build-type=release \
--bundle-arrow-cpp \
bdist_wheel

# pyarrow 24+ dropped setup.py in favour of scikit-build-core via pyproject.toml.
# --no-isolation reuses build deps already installed in the image (scikit-build-core,
# cython>=3.1, libcst, setuptools_scm, numpy) instead of pip resolving them again.
python3 -m build --wheel . --no-isolation \
-C cmake.build-type=Release

pip3 install dist/pyarrow-*.whl -t /aws-sdk-pandas/dist/pyarrow_files

Expand Down
Loading