[TF] Update TF v2.16.1 (#9241)

* Update TF v2.16.1

* update  abseil-cpp 20230802.2

* update bazel-absl patch

* TF2.16: Apply abseil aarch64 patch

* Update py3-scipy

* Update cython as well

* Try updating py3-blosc2

* Update py3-tables

* Add symlink for cudnn_frontend_archive

* Update tensorflow.spec

* Remove more files with broken symbols

* Update keras

* Add missing keras dependencies

* Build fft in tensorflow

* Update tensorflow-sources.file

* Update tensorflow-xla-runtime recipe to link with libfft_wrapper

* Try fixing unit tests

* Revert "Try fixing unit tests"

This reverts commit cd1a82e.

* Revert "Update tensorflow-xla-runtime recipe to link with libfft_wrapper"

This reverts commit cd38c1d.

* Changes from review

* Update tensorflow-xla-runtime.spec

* Update tensorflow-xla-runtime.spec

* Update tensorflow-xla-runtime.spec

* Try packing libfft* into wheel

* Update tensroflow-xla-runtime recipe

* Update tensorflow-sources.spec

* Remove relocation for missing files

* [TF][el7]: patch to avoid undeclared va_list

* Fix optree recipe

* Update tensorflow-sources.file

---------

Co-authored-by: Ivan Razumov <[email protected]>
Co-authored-by: Ivan Razumov <[email protected]>

Author: smuzaffar

abseil-cpp.spec

@@ -1,8 +1,9 @@
-### RPM external abseil-cpp 20230125.3
+### RPM external abseil-cpp 20230802.2
 ## INCLUDE cpp-standard
 
 Source: https://github.com/abseil/abseil-cpp/archive/%{realversion}.tar.gz
 Source2: https://patch-diff.githubusercontent.com/raw/abseil/abseil-cpp/pull/1732.diff
+
 BuildRequires: cmake gmake
 
 %prep

bazel-absl.patch

@@ -8,20 +8,20 @@ index b896f1b..0646f73 100755
      "com_google_absl": {
 -        "archive": "20211102.0.tar.gz",
 -        "sha256": "dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4",
-+        "archive": "20230125.3.tar.gz",
-+        "sha256": "5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36",
++        "archive": "20230802.2.tar.gz",
++        "sha256": "7c11539617af1f332f0854a6fb21e296a1b29c27d03f23c7b49d4adefcd102cc",
          "urls": [
 -            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.tar.gz",
 -            "https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.tar.gz",
-+            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz",
-+            "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz",
++            "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/refs/tags/20230802.2.tar.gz",
++            "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.2.tar.gz",
          ],
          "used_in": [
              "additional_distfiles",
              "test_WORKSPACE_files",
          ],
 -        "strip_prefix": "abseil-cpp-20211102.0",
-+        "strip_prefix": "abseil-cpp-20230125.3",
++        "strip_prefix": "abseil-cpp-20230802.2",
      },
      "zstd-jni": {
          "archive": "v1.5.2-3.zip",

pip/blosc2.file

@@ -1,5 +1,2 @@
 Requires: py3-scikit-build py3-cython py3-numpy py3-msgpack py3-ndindex py3-py-cpuinfo py3-rich
 BuildRequires: cmake ninja
-
-%define PipPostPost \
-  %{relocateConfig}lib64/pkgconfig/blosc2.pc

pip/keras.file

@@ -1,3 +1,4 @@
 %define PipDownloadSourceType none
 Requires: py3-PyYAML py3-six
 Requires: py3-h5py py3-keras-applications py3-keras-preprocessing
+Requires: py3-absl-py py3-ml_dtypes py3-namex py3-optree py3-rich

pip/numpy.file

@@ -1,3 +1,4 @@
+BuildRequires: py3-meson-python
 Requires: py3-cython
 Requires: zlib OpenBLAS
 
@@ -28,8 +29,6 @@ EOF
   ln -s ${numpy_core} %{i}/c-api/core
 
 %define PipPostPost \
-  %{relocateConfig}lib/python*/site-packages/numpy/__config__.py \
-  %{relocateConfig}lib/python*/site-packages/numpy/distutils/__config__.py \
-  %{relocateConfig}lib/python*/site-packages/numpy/distutils/site.cfg
+  %{relocateConfig}lib/python*/site-packages/numpy/__config__.py
 
 %define PipPostBuildPy rm -f %{i}/bin/f2py

pip/optree.file

@@ -0,0 +1,2 @@
+BuildRequires: cmake
+Requires: py3-typing-extensions

pip/requirements.txt

@@ -45,7 +45,7 @@ beautifulsoup4==4.12.3
 beniget==0.4.1
 bleach==6.1.0
 # NO_AUTO_UPDATE: needs Python 3.10
-blosc2==2.2.0
+blosc2==2.5.1
 bokeh==3.4.1
 boost-histogram==1.4.1
 bottle==0.12.25
@@ -84,7 +84,7 @@ cryptography==42.0.5
 cx-Oracle==8.3.0
 cycler==0.12.1
 # NO_AUTO_UPDATE: numpy 1.24.3 doesn't support cython 3.x
-cython==0.29.35
+cython==3.0.10
 dask==2024.4.2
 dask-awkward==2024.3.0
 decorator==5.1.1
@@ -185,8 +185,7 @@ jupyter-server-mathjax==0.2.6
 jupyter-server-terminals==0.5.3
 jupyterlab==4.1.8
 jupyterlab-widgets==3.0.10
-#NO_AUTO_UPDATE: needs newer TF
-keras==2.15.0
+keras==3.5.0
 keras-applications==1.0.8
 keras-preprocessing==1.1.2
 keras2onnx==1.7.0
@@ -224,6 +223,7 @@ mplhep-data==0.0.3
 mpmath==1.3.0
 msgpack==1.0.8
 multidict==6.0.5
+namex==0.0.8
 nbclassic==1.0.0
 nbclient==0.10.0
 nbconvert==7.16.4
@@ -244,12 +244,13 @@ nvidia-ml-py==12.550.52
 numexpr==2.8.4
 # setuptools version <64 is needed by numpy: https://github.com/pypa/setuptools/issues/3549
 # NO_AUTO_UPDATE: update together with tensorflow
-numpy==1.24.3
+numpy==1.26.4
 # NO_AUTO_UPDATE:1
 onnx==1.16.0
 onnxmltools==1.12.0
 onnxconverter-common==1.14.0
 oauthlib==3.2.2
+optree==0.12.1
 opt-einsum==3.3.0
 pkginfo==1.10.0
 packaging==24.0
@@ -349,7 +350,7 @@ scikit-build==0.16.7
 scikit-build-core==0.9.3
 scikit-learn==1.2.2
 scinum==2.1.0
-scipy==1.10.0
+scipy==1.12.0
 secretstorage==3.3.3
 semantic-version==2.10.0
 threadpoolctl==3.5.0
@@ -371,14 +372,13 @@ stack-data==0.6.3
 stevedore==5.2.0
 subprocess32==3.5.4
 sympy==1.12
-# NO_AUTO_UPDATE: Needs newer blosc2
-tables==3.8.0
+tables==3.9.0
 tblib==1.7.0
 tabulate==0.9.0
 tenacity==8.2.3
-#NO_AUTO_UPDATE:1: Force to use tensorflow 2.15x; this should match the version in tensorflow-sources.spec
-tensorflow==2.15.0
-tensorboard==2.16.2
+#NO_AUTO_UPDATE:1: Force to use tensorflow 2.16.1; this should match the version in tensorflow-sources.spec
+tensorflow==2.16.1
+tensorboard==2.17.0
 tensorflow-io-gcs-filesystem==0.37.0
 tensorflow-estimator==2.15.0
 tensorboard-data-server==0.7.2

tensorflow-sources.file

@@ -36,6 +36,13 @@ Source: git+https://github.com/%{github_user}/tensorflow.git?obj=%{branch}/%{tag
 %setup -q -n tensorflow-%{realversion}
 sed -i -e 's|lib/python[^/]*/site-packages/|lib/python%{cms_python3_major_minor_version}/site-packages/|'  third_party/systemlibs/pybind11.BUILD
 
+#EL7: Include stdarg.h before zlib.h to avoid 'va_list' has not been declared error.
+%if 0%{?rhel} == 7
+sed -i -e 's|#include <zlib.h>|#include <stdarg.h>\n#include <zlib.h>|' \
+  third_party/xla/third_party/tsl/tsl/lib/io/zlib_outputbuffer.h \
+  tensorflow/core/lib/png/png_io.cc
+%endif
+
 %build
 
 export PYTHON_BIN_PATH="$(which python3)"
@@ -144,6 +151,8 @@ export PYTHONPATH=$PYTHON3PATH
 
 ./configure
 # build tensorflow python targets
+bazel $BAZEL_OPTS @ducc//:fft
+bazel $BAZEL_OPTS @ducc//:fft_wrapper
 bazel $BAZEL_OPTS //tensorflow/tools/pip_package:build_pip_package
 bazel $BAZEL_OPTS //tensorflow:tensorflow
 bazel $BAZEL_OPTS //tensorflow:tensorflow_cc
@@ -180,10 +189,13 @@ mkdir -p $bindir $incdir $libdir
 
 # copy targets
 srcdir="$PWD/bazel-bin/tensorflow"
+srcdir_ext="$PWD/bazel-bin/external"
 
 cp -p $srcdir/libtensorflow*.so* $libdir/
 cp -p $srcdir/compiler/tf2xla/lib*.so* $libdir/
 
+cp -p ${srcdir_ext}/ducc/*.so $libdir/
+
 for l in tensorflow_cc tensorflow_framework tensorflow ; do
   # check if the actual lib exists
   [ -f $libdir/lib${l}.so.%{tfversion} ] || exit 1
@@ -225,7 +237,7 @@ popd
 #FIXME: Create missing externals links
 pushd bazel-tensorflow-%{realversion}/external
   srcdir=$(dirname $(readlink zlib))
-  for e in ducc farmhash_gpu_archive stablehlo ; do
+  for e in ducc farmhash_gpu_archive stablehlo cudnn_frontend_archive ; do
     echo "Check external link: $e"
     if [ -e ${srcdir}/$e ] ; then
       if [ ! -e $e ] ; then
@@ -236,6 +248,14 @@ pushd bazel-tensorflow-%{realversion}/external
   done
 popd
 
-# create the wheel file that is installed in py2-tensorflow
+# create the wheel file that is installed in py3-tensorflow
 export PYTHONPATH=$PYTHON3PATH
 bash -ex bazel-bin/tensorflow/tools/pip_package/build_pip_package %{i}
+
+mkdir %{i}/lib-xla-runtime
+cp $outdir/lib/libfft.so %{i}/lib-xla-runtime
+cp $outdir/lib/libfft_wrapper.so %{i}/lib-xla-runtime
+find $srcdir/.. -name 'libmutex.pic.a' -path '*k8-opt/bin/external/local_tsl*' -exec cp {} %{i}/lib-xla-runtime \;
+find $srcdir/.. -name 'libnsync_cpp.pic.a' -path '*k8-opt*' -exec cp {} %{i}/lib-xla-runtime \;
+[ -e %{i}/lib-xla-runtime/libmutex.pic.a ]
+[ -e %{i}/lib-xla-runtime/libnsync_cpp.pic.a ]

tensorflow-sources.spec

@@ -1,5 +1,5 @@
-### RPM external tensorflow-sources 2.15.0
-%define tag         3f5b9e20a2e5f678f79baffc9c2a59ac554053b2
+### RPM external tensorflow-sources 2.16.1
+%define tag         e21a182cfa2cfbb37125054201c2ccc12076d56b
 %define branch      cms/v%{realversion}
 %define github_user cms-externals
 %define build_type opt

tensorflow-xla-runtime.spec

@@ -8,11 +8,13 @@ Source99: scram-tools.file/tools/eigen/env
 Patch0: tensorflow-xla-runtime-absl
 
 Requires: eigen py3-tensorflow abseil-cpp
-BuildRequires: cmake
+BuildRequires: cmake tensorflow-sources
 
 %prep
 
 cp -r ${PY3_TENSORFLOW_ROOT}/lib/python%{cms_python3_major_minor_version}/site-packages/tensorflow .
+mkdir -p tensorflow/lib
+cp ${TENSORFLOW_SOURCES_ROOT}/lib-xla-runtime/* tensorflow/lib/
 %patch -p0
 
 %build
@@ -30,11 +32,13 @@ pushd tensorflow/xla_aot_runtime_src
     -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \
     -DCMAKE_CXX_STANDARD=%{cms_cxx_standard} \
     -DCMAKE_PREFIX_PATH=${ABSEIL_CPP_ROOT} \
+    -DCMAKE_SHARED_LINKER_FLAGS="-L../lib -lfft -lfft_wrapper -l:libmutex.pic.a -l:libnsync_cpp.pic.a" \
     -DBUILD_SHARED_LIBS=ON
   make %{makeprocesses}
 popd
 
 %install
 
 mkdir -p %{i}/lib
+mv tensorflow/lib/libfft*.so %{i}/lib/
 mv tensorflow/xla_aot_runtime_src/libtf_xla_runtime.so %{i}/lib/

tensorflow.spec

@@ -1,4 +1,4 @@
-### RPM external tensorflow 2.15.0
+### RPM external tensorflow 2.16.1
 %if "%{?vectorized_package:set}" != "set"
 %define source_package tensorflow-sources
 %else