[PATCH] Tensorflow Training 2.18 CVE Patch (#5024)

Jyothirmaikottu · web-flow · commit eecd71dbd11e · 2025-07-30T07:26:38.000-07:00
* build tensorflow 2.18 training sm

* build tensorflow 2.18 training ec2

* build tensorflow 2.19 training sm

* build tensorflow 2.19 training sm

* build tensorflow 2.18 training sm

* build tensorflow 2.18 training ec2

* build tensorflow 2.18 with opencv pinned version ec2

* build tensorflow 2.18 with opencv pinned version sm

* build tensorflow 2.18 sm

* build tensorflow 2.18 ec2

* build tensorflow 2.19 sm

* build tensorflow 2.18 sm

* build tensorflow 2.18 ec2

* build tensorflow 2.19 sm with open cv pinned

* retry build for tensorflow 2.18 sm

* retry build for tensorflow 2.18 ec2

* retry build for tensorflow 2.18 ec2

* retry build for tensorflow 2.18 sm

* retry build for tensorflow 2.18 sm

* retry build for tensorflow 2.18 ec2

* revert toml

* build 2.18 cve sagemaker image with protobuf 6

* build 2.18 cve sagemaker image with protobuf 6

* build 2.18 cve sagemaker image with protobuf 6

* build 2.18 cve ec2 image with protobuf 6

* build ec2

* build ec2

* build ec2

* run securtiy tests

* run securtiy tests

* rerun security

* build ec2

* build sm

* build ec2

* build ec2

* build sm

* rebuild sm

* rebuild sm

* remove version pins

* removed sm version pins

* rebuild sm

* sagemaker version pins

* tf build sm

* build sm

* build sm

* build sm

* build sm with only sagemaker

* build sm with only sagemaker

* build sm with only sagemaker

* repin versions

* typo fix sg build

* typo fix sg build

* build sm, split package installation

* build sm, make ssae 0.1.4

* rebuild sm with lesser protobuf version

* protobuf and numpy pin, build sm

* numpy pin, build sm

* rebuild with numpy

* rebuild with numpy

* rebuild sm

* fix numpy, rebuild

* fix installation command, rebuild

* build ec2 again

* revert toml

* test ec2 build

* revert toml

* comment out autpatch build
diff --git a/dlc_developer_config.toml b/dlc_developer_config.toml
@@ -72,15 +72,12 @@ ec2_benchmark_tests = false
 ### These tests are run in EC2 test jobs, so ec2_tests must be true if ec2_tests_on_heavy_instances is true.
 ### Off by default (set to false)
 ec2_tests_on_heavy_instances = false
-
 ### SM specific tests
 ### On by default
 sagemaker_local_tests = true
-
 ### Set enable_ipv6 = true to run tests with IPv6-enabled resources
 ### Off by default (set to false)
 enable_ipv6 = false
-
 ### Set the VPC name to be used for IPv6 testing, this variable is empty by default
 ### To create an IPv6-enabled VPC and its related resources:
 ### 1. Follow this AWS doc: https://docs.aws.amazon.com/vpc/latest/userguide/create-vpc.html#create-vpc-and-other-resources
diff --git a/tensorflow/training/buildspec-2-18-ec2.yml b/tensorflow/training/buildspec-2-18-ec2.yml
@@ -5,7 +5,7 @@ framework: &FRAMEWORK tensorflow
 version: &VERSION 2.18.0
 short_version: &SHORT_VERSION "2.18"
 arch_type: x86
-autopatch_build: "True"
+# autopatch_build: "True"
 
 repository_info:
   training_repository: &TRAINING_REPOSITORY
diff --git a/tensorflow/training/buildspec-2-18-sm.yml b/tensorflow/training/buildspec-2-18-sm.yml
@@ -5,7 +5,7 @@ framework: &FRAMEWORK tensorflow
 version: &VERSION 2.18.0
 short_version: &SHORT_VERSION "2.18"
 arch_type: x86
-autopatch_build: "True"
+# autopatch_build: "True"
 
 repository_info:
   training_repository: &TRAINING_REPOSITORY
diff --git a/tensorflow/training/docker/2.18/py3/Dockerfile.cpu b/tensorflow/training/docker/2.18/py3/Dockerfile.cpu
@@ -159,9 +159,8 @@ RUN ${PIP} install --no-cache-dir -U \
     absl-py \
     opencv-python \
     werkzeug \
-    psutil \
-    "protobuf<4"
-
+    psutil 
+    
 ADD https://raw.githubusercontent.com/aws/deep-learning-containers/master/src/deep_learning_container.py /usr/local/bin/deep_learning_container.py
 
 RUN chmod +x /usr/local/bin/deep_learning_container.py
@@ -189,7 +188,7 @@ ARG TF_URL
 RUN ${PIP} install --no-cache-dir -U \
     ${TF_URL} \
     "tensorflow-io==0.37.*" \
-    tensorflow-datasets
+    "tensorflow-datasets==4.9.7"
 
 RUN HOME_DIR=/root \
    && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \
@@ -261,26 +260,41 @@ RUN ${PIP} install --no-cache-dir -U \
  && ${PIP} install --no-cache-dir -U \
     ${TF_URL} \
     "tensorflow-io==0.37.*" \
-    tensorflow-datasets
+    "tensorflow-datasets==4.9.7"
 
 RUN $PYTHON -m pip install --no-cache-dir -U \
-    numba \
+    numba==0.61.0 \
     bokeh \
     imageio \
     opencv-python \
     plotly \
     seaborn \
-    shap
+    shap 
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    "sagemaker<3"
 
 RUN $PYTHON -m pip install --no-cache-dir -U \
-    "sagemaker<3" \
-    sagemaker-experiments==0.* \
-    sagemaker-tensorflow-training \
-    sagemaker-training \
-    "sagemaker-studio-analytics-extension<1" \
-    "sparkmagic<1" \
-    "sagemaker-studio-sparkmagic-lib<1" \
-    smclarify
+    sagemaker-experiments==0.1.45
+   
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sagemaker-tensorflow-training
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sagemaker-training
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sagemaker-studio-analytics-extension==0.1.4
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sagemaker-studio-sparkmagic-lib==0.2.0 
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sparkmagic==0.21.0 \
+    smclarify 
+
+#pin numpy version because of sagemaker-tensorflow-training dependency
+RUN $PYTHON -m pip install --no-cache-dir numpy==1.26.4
 
 # Remove python kernel installed by sparkmagic
 RUN /usr/local/bin/jupyter-kernelspec remove -f python3
diff --git a/tensorflow/training/docker/2.18/py3/Dockerfile.ec2.cpu.core_packages.json b/tensorflow/training/docker/2.18/py3/Dockerfile.ec2.cpu.core_packages.json
@@ -3,7 +3,7 @@
     "version_specifier": ">=1.33.13,<2"
   },
   "protobuf": {
-    "version_specifier": ">=3.20.3,<4"
+    "version_specifier": ">=4.21.12"
   },
   "pyyaml": {
     "version_specifier": ">=6.0,<6.1"
diff --git a/tensorflow/training/docker/2.18/py3/Dockerfile.ec2.cpu.py_scan_allowlist.json b/tensorflow/training/docker/2.18/py3/Dockerfile.ec2.cpu.py_scan_allowlist.json
@@ -0,0 +1,3 @@
+{
+    "77740": "[Package: protobuf] Affected versions of this package are vulnerable to a potential Denial of Service (DoS) attack due to unbounded recursion when parsing untrusted Protocol Buffers data. The pure-Python implementation fails to enforce recursion depth limits when processing recursive groups, recursive messages, or a series of SGROUP tags, leading to stack overflow conditions that can crash the application by exceeding Python's recursion limit."
+}
diff --git a/tensorflow/training/docker/2.18/py3/Dockerfile.sagemaker.cpu.core_packages.json b/tensorflow/training/docker/2.18/py3/Dockerfile.sagemaker.cpu.core_packages.json
@@ -6,7 +6,7 @@
     "version_specifier": "<3.0"
   },
   "protobuf": {
-    "version_specifier": ">=3.20.3,<4"
+    "version_specifier": ">=5.29.5"
   },
   "pyyaml": {
     "version_specifier": ">=6.0,<6.1"
@@ -18,7 +18,7 @@
     "version_specifier": ">=20.4.1,<21"
   },
   "sagemaker-training": {
-    "version_specifier": ">=4.7.4,<5"
+    "version_specifier": ">=5"
   },
   "sagemaker-studio-analytics-extension": {
     "version_specifier": "<1"
diff --git a/tensorflow/training/docker/2.18/py3/Dockerfile.sagemaker.cpu.py_scan_allowlist.json b/tensorflow/training/docker/2.18/py3/Dockerfile.sagemaker.cpu.py_scan_allowlist.json
@@ -0,0 +1,3 @@
+{
+    "77740": "[Package: protobuf] Affected versions of this package are vulnerable to a potential Denial of Service (DoS) attack due to unbounded recursion when parsing untrusted Protocol Buffers data. The pure-Python implementation fails to enforce recursion depth limits when processing recursive groups, recursive messages, or a series of SGROUP tags, leading to stack overflow conditions that can crash the application by exceeding Python's recursion limit."
+}
diff --git a/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.ec2.gpu.core_packages.json b/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.ec2.gpu.core_packages.json
@@ -3,7 +3,7 @@
     "version_specifier": ">=1.33.13,<2"
   },
   "protobuf": {
-    "version_specifier": ">=3.20.3,<4"
+    "version_specifier": ">=4.21.12"
   },
   "pyyaml": {
     "version_specifier": ">=6.0,<6.1"
diff --git a/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.ec2.gpu.py_scan_allowlist.json b/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.ec2.gpu.py_scan_allowlist.json
@@ -0,0 +1,3 @@
+{
+    "77740": "[Package: protobuf] Affected versions of this package are vulnerable to a potential Denial of Service (DoS) attack due to unbounded recursion when parsing untrusted Protocol Buffers data. The pure-Python implementation fails to enforce recursion depth limits when processing recursive groups, recursive messages, or a series of SGROUP tags, leading to stack overflow conditions that can crash the application by exceeding Python's recursion limit."
+}
diff --git a/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.gpu b/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.gpu
@@ -221,9 +221,8 @@ RUN ${PIP} install --no-cache-dir -U \
    absl-py \
    opencv-python \
    werkzeug \
-   urllib3 \
-   "protobuf<4"
-
+   urllib3 
+   
 # Install AWS OFI NCCL plug-in
 RUN apt-get update && apt-get install -y \
     autoconf \
@@ -246,6 +245,20 @@ RUN mkdir /tmp/efa-ofi-nccl \
  && make install \
  && rm -rf /tmp/efa-ofi-nccl
 
+# patch nvjpeg
+RUN mkdir -p /tmp/nvjpeg \
+&& cd /tmp/nvjpeg \
+&& wget https://developer.download.nvidia.com/compute/cuda/redist/libnvjpeg/linux-x86_64/libnvjpeg-linux-x86_64-12.4.0.76-archive.tar.xz \
+&& tar -xvf libnvjpeg-linux-x86_64-12.4.0.76-archive.tar.xz \
+&& rm -rf /usr/local/cuda/targets/x86_64-linux/lib/libnvjpeg* \
+&& rm -rf /usr/local/cuda/targets/x86_64-linux/include/nvjpeg.h \
+&& cp libnvjpeg-linux-x86_64-12.4.0.76-archive/lib/libnvjpeg* /usr/local/cuda/targets/x86_64-linux/lib/ \
+&& cp libnvjpeg-linux-x86_64-12.4.0.76-archive/include/* /usr/local/cuda/targets/x86_64-linux/include/ \
+&& rm -rf /tmp/nvjpeg \
+# patch cuobjdump and nvdisasm
+&& rm -rf /usr/local/cuda/bin/cuobjdump* \
+&& rm -rf /usr/local/cuda/bin/nvdisasm* 
+
 # Allow OpenSSH to talk to containers without asking for confirmation
 RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \
    && echo "    StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \
@@ -282,7 +295,7 @@ ARG TF_URL
 RUN ${PIP} install --no-cache-dir -U \
    ${TF_URL} \
    "tensorflow-io==0.37.*" \
-   tensorflow-datasets
+   "tensorflow-datasets==4.9.7"
 
 RUN HOME_DIR=/root \
    && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \
@@ -351,26 +364,41 @@ RUN ${PIP} install --no-cache-dir -U \
  && ${PIP} install --no-cache-dir -U \
     ${TF_URL} \
     "tensorflow-io==0.37.*" \
-    tensorflow-datasets
+    "tensorflow-datasets==4.9.7"
 
 RUN $PYTHON -m pip install --no-cache-dir -U \
-    numba \
+    numba==0.61.0 \
     bokeh \
     imageio \
     opencv-python \
     plotly \
     seaborn \
-    shap
+    shap 
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    "sagemaker<3"
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sagemaker-experiments==0.1.45
+   
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sagemaker-tensorflow-training
 
 RUN $PYTHON -m pip install --no-cache-dir -U \
-    "sagemaker<3" \
-    sagemaker-experiments==0.* \
-    sagemaker-tensorflow-training \
-    sagemaker-training \
-    "sagemaker-studio-analytics-extension<1" \
-    "sparkmagic<1" \
-    "sagemaker-studio-sparkmagic-lib<1" \
-    smclarify
+    sagemaker-training
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sagemaker-studio-analytics-extension==0.1.4
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sagemaker-studio-sparkmagic-lib==0.2.0 
+
+RUN $PYTHON -m pip install --no-cache-dir -U \
+    sparkmagic==0.21.0 \
+    smclarify 
+
+#pin numpy version because of sagemaker-tensorflow-training dependency
+RUN $PYTHON -m pip install --no-cache-dir numpy==1.26.4
 
 # install boost
 # tensorflow is compiled with --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=1"
diff --git a/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.sagemaker.gpu.core_packages.json b/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.sagemaker.gpu.core_packages.json
@@ -6,7 +6,7 @@
     "version_specifier": "<3.0"
   },
   "protobuf": {
-    "version_specifier": ">=3.20.3,<4"
+    "version_specifier": ">=5.29.5"
   },
   "pyyaml": {
     "version_specifier": ">=6.0,<6.1"
@@ -18,7 +18,7 @@
     "version_specifier": ">=20.4.1,<21"
   },
   "sagemaker-training": {
-    "version_specifier": ">=4.7.4,<5"
+    "version_specifier": ">=5"
   },
   "sagemaker-studio-analytics-extension": {
     "version_specifier": "<1"
diff --git a/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.sagemaker.gpu.py_scan_allowlist.json b/tensorflow/training/docker/2.18/py3/cu125/Dockerfile.sagemaker.gpu.py_scan_allowlist.json
@@ -0,0 +1,3 @@
+{
+    "77740": "[Package: protobuf] Affected versions of this package are vulnerable to a potential Denial of Service (DoS) attack due to unbounded recursion when parsing untrusted Protocol Buffers data. The pure-Python implementation fails to enforce recursion depth limits when processing recursive groups, recursive messages, or a series of SGROUP tags, leading to stack overflow conditions that can crash the application by exceeding Python's recursion limit."
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+{`
	`2`	`+ "77740": "[Package: protobuf] Affected versions of this package are vulnerable to a potential Denial of Service (DoS) attack due to unbounded recursion when parsing untrusted Protocol Buffers data. The pure-Python implementation fails to enforce recursion depth limits when processing recursive groups, recursive messages, or a series of SGROUP tags, leading to stack overflow conditions that can crash the application by exceeding Python's recursion limit."`
	`3`	`+}`