New feature: Add AlmaLinux8 Dockerfile with updated ML toolchain (TensorFlow 2.17 / CUDA 12.3)

Dockerfile-manylinux_2_28_x86_64

@@ -0,0 +1,78 @@
+FROM quay.io/pypa/manylinux_2_28_x86_64:latest
+
+COPY build_scripts/build_tools.sh build_scripts/build_3rdparty_static.sh build_scripts/install_cuda.sh build_scripts/uninstall_cuda.sh ./
+
+# The manylinux2014 image is intended to build essentia-tensorflow wheels
+# To build TensorFlow we additionally need Bazel and NumPy.
+ENV WITH_TENSORFLOW=true
+ENV PYTHON_BIN_PATH=/opt/python/cp310-cp310/bin/python3
+ENV PYTHONPATH=/usr/local/lib/python3.10/site-packages
+
+# Install cmake3 dependencies
+RUN yum install -y vim openssl-devel 	\
+	&& yum clean all 		\
+	&& rm -rf /var/cache/yum
+
+RUN /build_tools.sh && rm -r /build_tools.sh
+
+ENV PATH=/usr/lib64/qt4/bin:/usr/local/cmake-3.16.9/bin:${PATH}
+ENV PATH=/opt/python/cp310-cp310/bin/:${PATH}
+
+ENV PREFIX=/usr/local
+ENV PKG_CONFIG_PATH=${PREFIX}/lib/pkgconfig
+ENV LD_LIBRARY_PATH=${PREFIX}/lib:/usr/local/lib64:${LD_LIBRARY_PATH}
+ENV ESSENTIA_3RDPARTY_VERSION=v2.1_beta5-1421-g0ad01f29
+
+# The only known alternative to the interactive TensorFlow configuration is
+# via environment variables:
+# https://github.com/tensorflow/tensorflow/issues/8527#issuecomment-289272898
+ENV USE_DEFAULT_PYTHON_LIB_PATH=1
+
+ENV BAZEL_LINKLIBS=-l%:libstdc++.a
+
+ENV TF_NEED_JEMALLOC=1
+ENV TF_NEED_GCP=0
+ENV TF_NEED_HDFS=0
+ENV TF_ENABLE_XLA=0
+ENV TF_NEED_OPENCL=0
+ENV TF_CUDA_CLANG=0
+
+# TensorFlow CUDA & CuDNN versions intended for TensorFlow 2.5
+# For future updates check the GPU compatibility chart:
+# https://www.tensorflow.org/install/source#gpu
+ENV TF_NEED_CUDA=1
+ENV TF_CUDA_VERSION=12.3
+ENV TF_CUDNN_VERSION=9
+ENV CUDA_TOOLKIT_PATH=/usr/local/cuda
+ENV CUDNN_INSTALL_PATH=/usr
+
+ENV CC=/usr/bin/gcc
+ENV CXX=/usr/bin/g++
+
+# The compute capabilities define which GPUs can be used:
+# https://developer.nvidia.com/cuda-gpus#compute
+# Supporting more versions increases the library size, so
+# for the moment it is set to a conservative number that
+# covers some of the most popular deep learning GPUs:
+# 7.5: Geforce RTX 2080 (Ti)
+# 8.0: Geforce RTX 3090 - 3080 (Ti)
+# 8.6: Geforce RTX 30XX
+# 8.9: Geforce RTX 4080
+ENV TF_CUDA_COMPUTE_CAPABILITIES=7.5,8.0,8.6,8.9
+ENV TF_CUDA_PATHS=/usr/local/cuda,/usr
+ENV PATH=$CUDA_TOOLKIT_PATH/bin:$PATH
+
+# TensorFlow 2.17.0 only supports GCC 11 (GCC 14 by default)
+# CUDA 12.3 supports GCC ≤ 11 officially
+RUN yum -y install gcc-toolset-11 \
+	&& yum clean all 		\
+	&& rm -rf /var/cache/yum
+
+ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH
+ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-11/root/usr/lib64:$LD_LIBRARY_PATH
+
+# Install CUDA and Essentia dependencies
+RUN /install_cuda.sh && /build_3rdparty_static.sh && /uninstall_cuda.sh \
+  && rm -r /install_cuda.sh /build_3rdparty_static.sh /uninstall_cuda.sh
+
+CMD ["/bin/bash"]

build_scripts/build_3rdparty_static.sh

@@ -1,7 +1,7 @@
 set -e -x
 
 # Check out a particular commit for building dependencies
-curl -SLO https://github.com/MTG/essentia/archive/$ESSENTIA_3RDPARTY_VERSION.zip
+curl -SLO https://github.com/xaviliz/essentia/archive/$ESSENTIA_3RDPARTY_VERSION.zip
 unzip $ESSENTIA_3RDPARTY_VERSION.zip
 cd essentia-*/
 
@@ -10,7 +10,8 @@ if [[ "${WITH_TENSORFLOW}" == "true" ]]; then
 fi
 
 # Install dependencies to /usr/local; force --static flag to pickup private libraries for Qt
-PKGCONFIG="/usr/bin/pkg-config --static" ./packaging/build_3rdparty_static_debian.sh --with-gaia ${with_tensorflow}
+#! Gaia cannot be installed with Qt4 with cmake3, it needs to update to Qt5 first
+PKGCONFIG="/usr/bin/pkg-config --static" ./packaging/build_3rdparty_static_debian.sh ${with_tensorflow}
 
 cd ..
 rm -r essentia-* $ESSENTIA_3RDPARTY_VERSION.zip

build_scripts/build_tools.sh

@@ -60,7 +60,11 @@ done
 
 # Gaia's waf build script requires qt4 tools (qmake, uic, ...),
 # but they aren't really used. We should get rid of them in the future.
-yum -y install qt4-devel
+
+# There is no qt4-devel package in quay.io/pypa/manylinux_2_28_x86_64, is based on AlmaLinux 8
+# but it could be updated to qt5
+yum -y install qt5-qtbase-devel
+yum clean all && rm -rf /var/cache/yum
 
 if [[ "${WITH_TENSORFLOW}" == "true" ]]; then
   # Bazelisk is a wrapper for Bazel that downloads the correct version for
@@ -70,6 +74,14 @@ if [[ "${WITH_TENSORFLOW}" == "true" ]]; then
   yum install -y nodejs
   npm install -g @bazel/bazelisk
 
+  # Bazelisk automatically downloads the latest Bazel
+  # unless explicitly tell it which version to use
+  ln -s /usr/bin/bazelisk /usr/local/bin/bazel
+  export USE_BAZEL_VERSION=6.5.0
+
+  echo 'export USE_BAZEL_VERSION=6.5.0' >>~/.bashrc
+  source ~/.bashrc
+
   # NumPy is required by Bazel
   ${PYTHON_BIN_PATH} -m pip install numpy
 fi

build_scripts/install_cuda.sh

@@ -1,22 +1,64 @@
-set -e -x
+#!/bin/bash
+set -euo pipefail
 
-# Install CUDA and CuDNN so that TensorFlow can be built with GPU support.
-CUDA_VERSION=11.2.2-1
-CUDNN_VERSION=8.1.0.77-1
+CUDA_VERSION=12-3
+CUDA_MAJOR=12.3
 
-# First download the latest Nvidia CUDA from official repository and install CUDA
-yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo
-yum install -y cuda-${CUDA_VERSION}
+# ----------------------------
+# Add NVIDIA CUDA RHEL8 repo
+# ----------------------------
+curl -fsSL \
+  https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo \
+  -o /etc/yum.repos.d/cuda.repo
 
-# Install CuDNN and put it in the CUDA path
-curl -SLO https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/libcudnn8-${CUDNN_VERSION}.cuda${CUDA_VERSION::4}.x86_64.rpm
-curl -SLO https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/libcudnn8-devel-${CUDNN_VERSION}.cuda${CUDA_VERSION::4}.x86_64.rpm
-rpm -i libcudnn8-${CUDNN_VERSION}.cuda${CUDA_VERSION::4}.x86_64.rpm
-rpm -i libcudnn8-devel-${CUDNN_VERSION}.cuda${CUDA_VERSION::4}.x86_64.rpm
+yum clean all
+yum makecache
 
-cp -P /usr/include/cudnn* /usr/local/cuda/include
-cp -P /usr/lib64/libcudnn* /usr/local/cuda/lib64/
+# ----------------------------
+# Install minimal CUDA 12.3 toolkit
+# (no drivers inside container)
+# ----------------------------
+yum -y install \
+  cuda-nvcc-${CUDA_VERSION} \
+  cuda-cudart-devel-${CUDA_VERSION} \
+  cuda-libraries-devel-${CUDA_VERSION} \
+  cuda-toolkit-${CUDA_VERSION} \
+  --exclude=cuda-drivers
 
-# Clean Up
-rm libcudnn8-${CUDNN_VERSION}.cuda${CUDA_VERSION::4}.x86_64.rpm
-rm libcudnn8-devel-${CUDNN_VERSION}.cuda${CUDA_VERSION::4}.x86_64.rpm
+# ----------------------------
+# Install cuDNN 9.x for CUDA 12.3
+# (TensorFlow 2.17 requirement)
+# ----------------------------
+yum -y install \
+  libcudnn9-cuda-12 \
+  libcudnn9-devel-cuda-12
+
+# ----------------------------
+# Cleanup yum cache
+# ----------------------------
+yum clean all
+rm -rf /var/cache/yum /var/cache/dnf
+
+# ----------------------------
+# Environment setup
+# ----------------------------
+export CUDA_HOME=/usr/local/cuda-${CUDA_MAJOR}
+export PATH=${CUDA_HOME}/bin:${PATH}
+export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
+
+# Optional: symlink for compatibility
+ln -sfn /usr/local/cuda-${CUDA_MAJOR} /usr/local/cuda
+
+# ----------------------------
+# Verify installation
+# ----------------------------
+echo "=== NVCC ==="
+nvcc --version
+
+echo "=== cuDNN ==="
+ldconfig -p | grep cudnn || echo "cuDNN not visible in linker cache"
+
+# ----------------------------
+# Final cleanup to reduce image size
+# ----------------------------
+rm -rf /root/.cache /tmp/* /var/tmp/*

build_scripts/uninstall_cuda.sh

@@ -1,9 +1,17 @@
 set -e -x
 
 # Uninstall CUDA and cuDNN so after TensorFlow is built.
-rpm -e libcudnn8-devel
-rpm -e libcudnn8
-yum autoremove -y cuda
+yum remove -y 'libcudnn9-devel-cuda-12'
+yum remove -y \
+  cuda-nvcc-* \
+  cuda-cudart-devel-* \
+  cuda-libraries-devel-* \
+  cuda-toolkit-*
+
+# Autoclean unused deps
+yum autoremove -y
+yum clean all
+rm -rf /var/cache/yum /var/Cache/dnf
 
 # Remove missing cuDNN libs
 rm -rf /usr/local/cuda-*