Patch 2025 02 07 (#4541)

* remove emacs

* Update ['dlc_developer_config.toml']

dlc_developer_config.toml:
{   'build': {   'build_frameworks': ['huggingface_pytorch'],
                 'build_inference': False,
                 'build_training': True},
    'buildspec_override': {   'dlc-pr-huggingface-pytorch-training': 'huggingface/pytorch/training/buildspec.yml'},
    'dev': {   'arm64_mode': False,
               'deep_canary_mode': False,
               'graviton_mode': False,
               'neuronx_mode': False},
    'test': {   'ec2_tests': True,
                'ecs_tests': True,
                'eks_tests': True,
                'sagemaker_local_tests': True,
                'sagemaker_remote_tests': True,
                'sanity_tests': True,
                'security_tests': True}}

* use RUN

* try using apt-get remove

* just remove emacs

* combine the calls

* bump transformers

* bump pip too

* just fix emacs for now

* should test transformers w/ v2 version actually

* upgrade pip

* bump ninja to 1.11.1.2

* remove test since we upgraded pip

* try setting ninja version

* bring test back

* remove emacs test

* added 2.1 change by accident, reverting

* fix flake8, bump release training version

* Revert "Update ['dlc_developer_config.toml']"

This reverts commit 9dfa272.

Author: timkuo-amazon

huggingface/pytorch/training/buildspec.yml

@@ -24,7 +24,7 @@ images:
     tag_python_version: &TAG_PYTHON_VERSION py311
     cuda_version: &CUDA_VERSION cu121
     os_version: &OS_VERSION ubuntu20.04
-    transformers_version: &TRANSFORMERS_VERSION 4.46.1
+    transformers_version: &TRANSFORMERS_VERSION 4.48.0
     datasets_version: &DATASETS_VERSION 3.1.0
     tag: !join [ *VERSION, '-', 'transformers', *TRANSFORMERS_VERSION, '-', *DEVICE_TYPE, '-', *TAG_PYTHON_VERSION, '-',
                  *CUDA_VERSION, '-', *OS_VERSION ]

huggingface/pytorch/training/docker/2.3/py3/cu121/Dockerfile.gpu

@@ -4,8 +4,11 @@
 # docker image region us-west-2
 FROM 763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:2.3.0-gpu-py311-cu121-ubuntu20.04-sagemaker
 
+RUN apt-get remove -y --purge emacs && \
+apt-get autoremove -y
+
 LABEL maintainer="Amazon AI"
-LABEL dlc_major_version="1"
+LABEL dlc_major_version="2"
 
 # version args
 ARG TRANSFORMERS_VERSION
@@ -17,10 +20,11 @@ ARG ACCELERATE_VERSION=1.1.0
 ARG TRL_VERSION=0.11.4
 ARG PEFT_VERSION=0.13.2
 ARG FLASH_ATTN_VERSION=2.6.3
+ARG NINJA_VERSION=1.11.1
 ARG PYTHON=python3
 
 # TODO: Remove when the base image is updated
-RUN pip install --upgrade "pip==24.1.2" \
+RUN pip install --upgrade pip \
  && pip uninstall -y transformer-engine flash-attn pyarrow cryptography \
  && pip install --no-cache-dir -U pyarrow cryptography pyopenssl Pillow \
  && pip --no-cache-dir install --upgrade wheel setuptools \
@@ -38,6 +42,7 @@ RUN pip install --no-cache-dir \
 	bitsandbytes \
 	evaluate==${EVALUATE_VERSION} \
 	accelerate==${ACCELERATE_VERSION} \
+	ninja==${NINJA_VERSION} \
 	trl==${TRL_VERSION} \
 	peft==${PEFT_VERSION} \
 	flash-attn==${FLASH_ATTN_VERSION}

release_images_training.yml

@@ -55,7 +55,7 @@ release_images:
   5:
     framework: "huggingface_pytorch"
     version: "2.3.0"
-    hf_transformers: "4.46.1"
+    hf_transformers: "4.48.0"
     arch_type: "x86"
     training:
       device_types: ["gpu"]

test/dlc_tests/sanity/test_utility_installation.py

@@ -186,24 +186,6 @@ def test_boto3(mxnet_inference):
     test_utils.run_cmd_on_container(container_name, ctx, "import boto3", executable="python")
 
 
-@pytest.mark.usefixtures("sagemaker", "functionality_sanity")
-@pytest.mark.model("N/A")
-@pytest.mark.integration("emacs")
-def test_emacs(image):
-    """
-    Ensure that emacs is installed on every image
-
-    :param image: ECR image URI
-    """
-    ctx = Context()
-    container_name = test_utils.get_container_name("emacs", image)
-    test_utils.start_container(container_name, image, ctx)
-
-    # Make sure the following emacs sanity tests exit with code 0
-    test_utils.run_cmd_on_container(container_name, ctx, "which emacs")
-    test_utils.run_cmd_on_container(container_name, ctx, "emacs -version")
-
-
 @pytest.mark.usefixtures("sagemaker_only", "functionality_sanity")
 @pytest.mark.model("N/A")
 @pytest.mark.integration("sagemaker_studio_analytics_extension")