Merge pull request #264 from jinyan-li1/py312-upgrade-clean

Upgrade scikit-learn container to Python 3.12

Author: jinyan-li1

.gitignore

@@ -7,3 +7,5 @@ src/sagemaker_sklearn_container.egg-info/
 __pycache__
 .coverage*
 .mypy_cache/
+Dockerfile.test
+test.parquet

docker/1.4-2-py312/base/Dockerfile.cpu

@@ -0,0 +1,91 @@
+ARG UBUNTU_VERSION=24.04
+ARG UBUNTU_IMAGE_DIGEST=b359f1067efa76f37863778f7b6d0e8d911e3ee8efa807ad01fbf5dc1ef9006b
+
+# Build stage for SQLite compilation
+FROM ubuntu:${UBUNTU_VERSION}@sha256:${UBUNTU_IMAGE_DIGEST} as sqlite-builder
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential wget ca-certificates && \
+    cd /tmp && \
+    wget https://www.sqlite.org/2025/sqlite-autoconf-3500200.tar.gz && \
+    tar xzf sqlite-autoconf-3500200.tar.gz && \
+    cd sqlite-autoconf-3500200 && \
+    ./configure --prefix=/usr/local && \
+    make && \
+    make install && \
+    ldconfig && \
+    cd / && \
+    rm -rf /tmp/sqlite-autoconf-3500200 /tmp/sqlite-autoconf-3500200.tar.gz && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Main image
+FROM ubuntu:${UBUNTU_VERSION}@sha256:${UBUNTU_IMAGE_DIGEST}
+
+ARG PYTHON_VERSION=3.12
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install runtime dependencies only
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install --no-install-recommends \
+        curl git jq libatlas-base-dev nginx openjdk-8-jdk-headless unzip wget expat tzdata apparmor \
+        libgstreamer1.0-0 libxml2 libsqlite3-0 software-properties-common ca-certificates lsb-release \
+        build-essential linux-libc-dev && \
+    # Add Apache Arrow repository for runtime libraries only
+    wget https://packages.apache.org/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+    apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+    apt-get update && \
+    apt-get install -y -V libarrow-dev=17.0.0-1 libarrow-dataset-dev=17.0.0-1 libparquet-dev=17.0.0-1 libarrow-acero-dev=17.0.0-1 && \
+    # Add deadsnakes PPA for Python 3.12
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get -y install --no-install-recommends \
+        python3.12 python3.12-dev && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 && \
+    # Create python symlink for backward compatibility
+    ln -sf /usr/bin/python3 /usr/bin/python && \
+    curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --break-system-packages && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime && \
+    dpkg-reconfigure --frontend noninteractive tzdata
+
+# Install uv for fast Python package management
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \
+    mv /root/.local/bin/uv /usr/local/bin/uv
+
+ENV PATH=/usr/local/bin:${PATH}
+ENV PIP_ROOT_USER_ACTION=ignore
+
+# Copy compiled SQLite from builder stage
+COPY --from=sqlite-builder /usr/local/bin/sqlite3 /usr/local/bin/sqlite3
+COPY --from=sqlite-builder /usr/local/lib/libsqlite3.* /usr/local/lib/
+COPY --from=sqlite-builder /usr/local/include/sqlite3*.h /usr/local/include/
+
+# Update library cache and ensure /usr/local/bin is in PATH
+RUN ldconfig && \
+    echo "/usr/local/lib" > /etc/ld.so.conf.d/sqlite3.conf && \
+    ldconfig
+
+ENV PATH="/usr/local/bin:${PATH}"
+
+# This command will check the version and print it to the build logs
+RUN sqlite3 --version
+
+# Install awscli
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    ./aws/install && \
+    rm -r aws awscliv2.zip
+
+# Python won't try to write .pyc or .pyo files on the import of source modules
+# Force stdin, stdout and stderr to be totally unbuffered. Good for logging
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONIOENCODING=UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+# Install core scientific packages with exact versions
+RUN uv pip install --system --no-cache --break-system-packages \
+    numpy==2.1.0 \
+    scikit-learn==1.4.2 \
+    pyarrow==17.0.0

docker/1.4-2/extension/Dockerfile.cpu …ker/1.4-2-py312/extension/Dockerfile.cpudocker/1.4-2/extension/Dockerfile.cpu renamed to docker/1.4-2-py312/extension/Dockerfile.cpu docker/1.4-2/extension/Dockerfile.cpu renamed to docker/1.4-2-py312/extension/Dockerfile.cpu

@@ -1,4 +1,4 @@
-FROM preprod-sklearn:1.4-2
+FROM preprod-sklearn:1.4-2-py312
 
 RUN pip freeze | grep -q 'scikit-learn==1.4.2'; \
 	if [ $? -eq 0 ]; \

docker/1.4-2/extension/README.md docker/1.4-2-py312/extension/README.mddocker/1.4-2/extension/README.md renamed to docker/1.4-2-py312/extension/README.md

@@ -1,24 +1,32 @@
-FROM sklearn-base:1.4-2
-ENV SAGEMAKER_SKLEARN_VERSION 1.4-2
+FROM sklearn-base:1.4-2-py312
+ENV SAGEMAKER_SKLEARN_VERSION 1.4-2-py312
 ENV PIP_ROOT_USER_ACTION=ignore
 
 LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true
 
 # Install remaining packages via pip
 COPY requirements.txt /requirements.txt
-RUN uv pip install --system -r /requirements.txt && \
+RUN uv pip install --system --break-system-packages -r /requirements.txt && \
     rm /requirements.txt
 
-# Fix Python 3.10 compatibility for sagemaker-containers
-RUN python3 -c "import sys; import os; site_packages = '/usr/local/lib/python3.10/dist-packages'; mapping_file = os.path.join(site_packages, 'sagemaker_containers/_mapping.py'); exec('if os.path.exists(mapping_file):\\n    with open(mapping_file, \"r\") as f:\\n        content = f.read()\\n    content = content.replace(\"collections.Mapping\", \"collections.abc.Mapping\")\\n    with open(mapping_file, \"w\") as f:\\n        f.write(content)')"
+# Fix Python 3.12 compatibility for sagemaker-containers
+RUN python3 -c "import sys; import os; site_packages = '/usr/local/lib/python3.12/dist-packages'; mapping_file = os.path.join(site_packages, 'sagemaker_containers/_mapping.py'); exec('if os.path.exists(mapping_file):\\n    with open(mapping_file, \"r\") as f:\\n        content = f.read()\\n    content = content.replace(\"collections.Mapping\", \"collections.abc.Mapping\")\\n    with open(mapping_file, \"w\") as f:\\n        f.write(content)')"
 
 COPY dist/sagemaker_sklearn_container-2.0-py3-none-any.whl /sagemaker_sklearn_container-2.0-py3-none-any.whl
-RUN uv pip install --system --no-cache /sagemaker_sklearn_container-2.0-py3-none-any.whl && \
+RUN uv pip install --system --no-cache --break-system-packages /sagemaker_sklearn_container-2.0-py3-none-any.whl && \
     rm /sagemaker_sklearn_container-2.0-py3-none-any.whl
 
+# Force upgrade six to 1.16.0 after all packages are installed to ensure six.moves compatibility
+RUN uv pip install --system --break-system-packages --upgrade --force-reinstall six==1.16.0
+
 # CWE-502: Patch sagemaker-inference decoder to disable pickle deserialization
 COPY docker/$SAGEMAKER_SKLEARN_VERSION/resources/patches/decoder.py \
-     /usr/local/lib/python3.10/dist-packages/sagemaker_inference/decoder.py
+     /usr/local/lib/python3.12/dist-packages/sagemaker_inference/decoder.py
+
+# Configure pip to always use --break-system-packages for Python 3.12
+RUN mkdir -p /root/.config/pip && \
+    echo "[global]" > /root/.config/pip/pip.conf && \
+    echo "break-system-packages = true" >> /root/.config/pip/pip.conf
 
 ENV SAGEMAKER_TRAINING_MODULE sagemaker_sklearn_container.training:main
 ENV SAGEMAKER_SERVING_MODULE sagemaker_sklearn_container.serving:main
@@ -60,4 +68,4 @@ EXPOSE 8080
 ENV TEMP=/home/model-server/tmp
 
 # Required label for multi-model loading
-LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
+LABEL com.amazonaws.sagemaker.capabilities.multi-models=true

docker/1.4-2/final/Dockerfile.cpu docker/1.4-2-py312/final/Dockerfile.cpudocker/1.4-2/final/Dockerfile.cpu renamed to docker/1.4-2-py312/final/Dockerfile.cpu

@@ -0,0 +1,96 @@
+# Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the 'License'). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the 'license' file accompanying this file. This file is
+# distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""This module contains functionality for converting various types of
+files and objects to NumPy arrays."""
+from __future__ import absolute_import
+
+import json
+from typing import Iterable  # noqa ignore=F401 imported but unused
+
+import numpy as np
+from six import BytesIO, StringIO
+
+from sagemaker_inference import content_types, errors
+
+
+def _json_to_numpy(string_like, dtype=None):  # type: (str) -> np.array
+    """Convert a JSON object to a numpy array.
+
+        Args:
+            string_like (str): JSON string.
+            dtype (dtype, optional):  Data type of the resulting array.
+                If None, the dtypes will be determined by the contents
+                of each column, individually. This argument can only be
+                used to 'upcast' the array.  For downcasting, use the
+                .astype(t) method.
+
+        Returns:
+            (np.array): numpy array
+        """
+    data = json.loads(string_like)
+    return np.array(data, dtype=dtype)
+
+
+def _csv_to_numpy(string_like, dtype=None):  # type: (str) -> np.array
+    """Convert a CSV object to a numpy array.
+
+    Args:
+        string_like (str): CSV string.
+        dtype (dtype, optional):  Data type of the resulting array. If None,
+            the dtypes will be determined by the contents of each column,
+            individually. This argument can only be used to 'upcast' the array.
+            For downcasting, use the .astype(t) method.
+
+    Returns:
+        (np.array): numpy array
+    """
+    stream = StringIO(string_like)
+    return np.genfromtxt(stream, dtype=dtype, delimiter=",")
+
+
+def _npy_to_numpy(npy_array):  # type: (object) -> np.array
+    """Convert a NPY array into numpy.
+
+    Args:
+        npy_array (npy array): to be converted to numpy array
+
+    Returns:
+        (np.array): converted numpy array.
+    """
+    stream = BytesIO(npy_array)
+    return np.load(stream, allow_pickle=False)
+
+
+_decoder_map = {
+    content_types.NPY: _npy_to_numpy,
+    content_types.CSV: _csv_to_numpy,
+    content_types.JSON: _json_to_numpy,
+}
+
+
+def decode(obj, content_type):
+    # type: (np.array or Iterable or int or float, str) -> np.array
+    """Decode an object to one of the default content types to a numpy array.
+
+    Args:
+        obj (object): to be decoded.
+        content_type (str): content type to be used.
+
+    Returns:
+        np.array: decoded object.
+    """
+    try:
+        decoder = _decoder_map[content_type]
+        return decoder(obj)
+    except KeyError:
+        raise errors.UnsupportedFormatError(content_type)