Skip to content

Commit 46e42a3

Browse files
sarckkfacebook-github-bot
sarckk
authored and
facebook-github-bot
committed
Update gcc version for FBGEMM install in CI (#2654)
Summary: TorchRec CI currently is failing with issues on incompatible GLIBCXX version. The cause is that FBGEMM now requires g++ 11.1+ for building binaries that reference GLIBCXX_3.4.29 (as of pytorch/pytorch#141035) As recommended in https://github.com/pytorch/FBGEMM/blob/main/.github/scripts/utils_build.bash and pytorch/FBGEMM#3423, install GCC using conda to control glibcxx version being used. Differential Revision: D67607624
1 parent 00d8ed2 commit 46e42a3

File tree

3 files changed

+72
-8
lines changed

3 files changed

+72
-8
lines changed

‎.github/workflows/docs.yml

+23-2
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ jobs:
3333
- name: Setup conda
3434
run: |
3535
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
36-
bash ~/miniconda.sh -b -p $HOME/miniconda
36+
bash ~/miniconda.sh -b -p $HOME/miniconda -u
37+
conda update -n base -c defaults -y conda
3738
- name: setup Path
3839
run: |
3940
echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
@@ -51,7 +52,27 @@ jobs:
5152
- name: Install gcc
5253
shell: bash
5354
run: |
54-
sudo apt-get install build-essential
55+
conda install -n build_binary -c conda-forge -y gxx_linux-64=11.4.0 sysroot_linux-64=2.17
56+
echo "[INSTALL] Setting the C/C++ compiler symlinks ..."
57+
cc_path=$(conda run -n build_binary printenv CC)
58+
cxx_path=$(conda run -n build_binary printenv CXX)
59+
ln -sf "${cc_path}" "$(dirname "$cc_path")/cc"
60+
ln -sf "${cc_path}" "$(dirname "$cc_path")/gcc"
61+
ln -sf "${cxx_path}" "$(dirname "$cxx_path")/c++"
62+
ln -sf "${cxx_path}" "$(dirname "$cxx_path")/g++"
63+
64+
conda_prefix=$(conda run -n build_binary printenv CONDA_PREFIX)
65+
echo "[TEST] Enumerating libstdc++.so files ..."
66+
all_libcxx_libs=$(find "${conda_prefix}/lib" -type f -name 'libstdc++.so*' -print | sort)
67+
for f in $all_libcxx_libs; do
68+
echo "$f";
69+
objdump -TC "$f" | grep GLIBCXX_ | sed 's/.*GLIBCXX_\([.0-9]*\).*/GLIBCXX_\1/g' | sort -Vu | cat
70+
echo ""
71+
done
72+
73+
echo "[TEST] Appending the Conda-installed libstdc++ to LD_PRELOAD ..."
74+
current_value=$(conda run -n build_binary printenv LD_PRELOAD)
75+
conda env config vars set -n build_binary LD_PRELOAD="${current_value:+${current_value}:}${all_libcxx_libs[0]}"
5576
- name: setup Path
5677
run: |
5778
echo /usr/local/bin >> $GITHUB_PATH

‎.github/workflows/release_build.yml

+24-3
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ jobs:
4848
run: |
4949
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
5050
bash ~/miniconda.sh -b -p $HOME/miniconda -u
51+
conda update -n base -c defaults -y conda
5152
- name: setup Path
5253
run: |
5354
echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
@@ -62,9 +63,29 @@ jobs:
6263
- name: check python version
6364
run: |
6465
conda run -n build_binary python --version
65-
- name: Install C/C++ compilers
66-
run: |
67-
sudo yum install -y gcc gcc-c++
66+
- name: Install gcc
67+
run: |
68+
conda install -n build_binary -c conda-forge -y gxx_linux-64=11.4.0 sysroot_linux-64=2.17
69+
echo "[INSTALL] Setting the C/C++ compiler symlinks ..."
70+
cc_path=$(conda run -n build_binary printenv CC)
71+
cxx_path=$(conda run -n build_binary printenv CXX)
72+
ln -sf "${cc_path}" "$(dirname "$cc_path")/cc"
73+
ln -sf "${cc_path}" "$(dirname "$cc_path")/gcc"
74+
ln -sf "${cxx_path}" "$(dirname "$cxx_path")/c++"
75+
ln -sf "${cxx_path}" "$(dirname "$cxx_path")/g++"
76+
77+
conda_prefix=$(conda run -n build_binary printenv CONDA_PREFIX)
78+
echo "[TEST] Enumerating libstdc++.so files ..."
79+
all_libcxx_libs=$(find "${conda_prefix}/lib" -type f -name 'libstdc++.so*' -print | sort)
80+
for f in $all_libcxx_libs; do
81+
echo "$f";
82+
objdump -TC "$f" | grep GLIBCXX_ | sed 's/.*GLIBCXX_\([.0-9]*\).*/GLIBCXX_\1/g' | sort -Vu | cat
83+
echo ""
84+
done
85+
86+
echo "[TEST] Appending the Conda-installed libstdc++ to LD_PRELOAD ..."
87+
current_value=$(conda run -n build_binary printenv LD_PRELOAD)
88+
conda env config vars set -n build_binary LD_PRELOAD="${current_value:+${current_value}:}${all_libcxx_libs[0]}"
6889
- name: Install PyTorch and CUDA
6990
shell: bash
7091
run: |

‎.github/workflows/unittest_ci_cpu.yml

+25-3
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,31 @@ jobs:
4545
conda info
4646
python --version
4747
conda run -n build_binary python --version
48+
49+
echo "[INSTALL] Installing gcc..."
50+
conda install -n build_binary -c conda-forge -y gxx_linux-64=11.4.0 sysroot_linux-64=2.17
51+
52+
echo "[INSTALL] Setting the C/C++ compiler symlinks ..."
53+
cc_path=$(conda run -n build_binary printenv CC)
54+
cxx_path=$(conda run -n build_binary printenv CXX)
55+
ln -sf "${cc_path}" "$(dirname "$cc_path")/cc"
56+
ln -sf "${cc_path}" "$(dirname "$cc_path")/gcc"
57+
ln -sf "${cxx_path}" "$(dirname "$cxx_path")/c++"
58+
ln -sf "${cxx_path}" "$(dirname "$cxx_path")/g++"
59+
60+
conda_prefix=$(conda run -n build_binary printenv CONDA_PREFIX)
61+
echo "[INSTALL] Enumerating libstdc++.so files ..."
62+
all_libcxx_libs=$(find "${conda_prefix}/lib" -type f -name 'libstdc++.so*' -print | sort)
63+
for f in $all_libcxx_libs; do
64+
echo "$f";
65+
objdump -TC "$f" | grep GLIBCXX_ | sed 's/.*GLIBCXX_\([.0-9]*\).*/GLIBCXX_\1/g' | sort -Vu | cat
66+
echo ""
67+
done
68+
69+
echo "[INSTALL] Appending the Conda-installed libstdc++ to LD_PRELOAD ..."
70+
current_value=$(conda run -n build_binary printenv LD_PRELOAD)
71+
conda env config vars set -n build_binary LD_PRELOAD="${current_value:+${current_value}:}${all_libcxx_libs[0]}"
72+
4873
conda run -n build_binary \
4974
pip install torch --index-url https://download.pytorch.org/whl/nightly/cpu
5075
conda run -n build_binary \
@@ -73,9 +98,6 @@ jobs:
7398
python -m pytest torchrec -v -s -W ignore::pytest.PytestCollectionWarning --continue-on-collection-errors \
7499
--ignore-glob=**/test_utils/
75100
echo "Starting C++ Tests"
76-
conda install -n build_binary -y gxx_linux-64
77-
conda run -n build_binary \
78-
x86_64-conda-linux-gnu-g++ --version
79101
conda install -n build_binary -c anaconda redis -y
80102
conda run -n build_binary redis-server --daemonize yes
81103
mkdir cpp-build

0 commit comments

Comments
 (0)