Skip to content

Commit 2ef53f0

Browse files
excelle08facebook-github-bot
excelle08
authored and
facebook-github-bot
committed
Make FeedSim dep versions env-overridable + add prebuilt libtorch-from-wheel path
Differential Revision: D112069862
1 parent 634bc47 commit 2ef53f0

4 files changed

Lines changed: 115 additions & 36 deletions

File tree

‎packages/feedsim/install_feedsim.sh‎

Lines changed: 53 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,20 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P)
1010
BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")"
1111
FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim"
1212
FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party"
13-
LIBTORCH_VERSION="2.8.0"
13+
LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}"
14+
# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel
15+
# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip.
16+
# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no
17+
# standalone zip); harmless for older versions. Default 1 pairs with the
18+
# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without
19+
# additional env overrides.
20+
LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}"
21+
# Dependency versions are env-overridable so experiments can bump them without
22+
# forking this script; defaults reproduce the v2 baseline exactly.
23+
JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}"
24+
LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}"
25+
# Export so the aarch64 sub-installer (dispatched below) inherits the pins.
26+
export LIBTORCH_VERSION LIBTORCH_FROM_WHEEL FEEDSIM_JEMALLOC_VERSION FEEDSIM_LIBEVENT_VERSION
1427
DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz"
1528
echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}"
1629

@@ -45,7 +58,7 @@ dnf install -y bc ninja-build flex bison git texinfo binutils-devel \
4558
libsodium-devel libunwind-devel bzip2-devel double-conversion-devel \
4659
libzstd-devel lz4-devel xz-devel snappy-devel libtool bzip2 openssl-devel \
4760
zlib-devel libdwarf libdwarf-devel libaio-devel libatomic patch jq \
48-
xxhash xxhash-devel unzip rsync liburing-devel
61+
xxhash xxhash-devel unzip rsync liburing-devel python3-pip
4962

5063
# Creates feedsim directory under benchmarks/
5164
mkdir -p "${BENCHPRESS_ROOT}/benchmarks/feedsim"
@@ -178,30 +191,30 @@ else
178191
fi
179192

180193
# Installing JEMalloc
181-
if ! [ -d "jemalloc-5.3.0" ]; then
182-
wget "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
183-
bunzip2 "jemalloc-5.3.0.tar.bz2"
184-
tar -xvf "jemalloc-5.3.0.tar"
185-
cd "jemalloc-5.3.0"
194+
if ! [ -d "jemalloc-${JEMALLOC_VERSION}" ]; then
195+
wget "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2"
196+
bunzip2 "jemalloc-${JEMALLOC_VERSION}.tar.bz2"
197+
tar -xvf "jemalloc-${JEMALLOC_VERSION}.tar"
198+
cd "jemalloc-${JEMALLOC_VERSION}"
186199
./configure --enable-prof --enable-prof-libunwind
187200
make -j"$(nproc)"
188201
make install
189202
cd ../
190203
else
191-
msg "[SKIPPED] jemalloc-5.3.0"
204+
msg "[SKIPPED] jemalloc-${JEMALLOC_VERSION}"
192205
fi
193206

194207
# Installing libevent
195-
if ! [ -d "libevent-2.1.12-stable" ]; then
196-
wget "https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz"
197-
tar -xzf "libevent-2.1.12-stable.tar.gz"
198-
cd "libevent-2.1.12-stable"
208+
if ! [ -d "libevent-${LIBEVENT_VERSION}" ]; then
209+
wget "https://github.com/libevent/libevent/releases/download/release-${LIBEVENT_VERSION}/libevent-${LIBEVENT_VERSION}.tar.gz"
210+
tar -xzf "libevent-${LIBEVENT_VERSION}.tar.gz"
211+
cd "libevent-${LIBEVENT_VERSION}"
199212
./configure
200213
make -j"$(nproc)"
201214
make install
202215
cd ../
203216
else
204-
msg "[SKIPPED] libevent-2.1.12-stable"
217+
msg "[SKIPPED] libevent-${LIBEVENT_VERSION}"
205218
fi
206219

207220
msg "Installing third-party dependencies ... DONE"
@@ -218,12 +231,33 @@ else
218231
fi
219232

220233
if ! [ -d "libtorch" ]; then
221-
msg "Downloading LibTorch ${LIBTORCH_VERSION}..."
222-
wget "${LIBTORCH_URL}" -O libtorch.zip
223-
msg "Extracting LibTorch..."
224-
unzip -q libtorch.zip
225-
rm libtorch.zip
226-
msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch"
234+
if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then
235+
# Extract LibTorch from the prebuilt torch CPU wheel. The wheel's
236+
# torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as
237+
# the standalone libtorch zip, so we just rename it to libtorch/.
238+
msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..."
239+
# pip on the box (3.9, or an internal stale mirror) can't see the cp310
240+
# 2.13 wheels, so resolve the wheel href straight from the PEP-503 index
241+
# and wget it. The C++ libtorch inside (torch/lib, torch/share/cmake) is
242+
# Python-version independent, so the cp310 wheel is fine for our C++ link.
243+
WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \
244+
| grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \
245+
| head -1)"
246+
[ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index"
247+
msg "Wheel: ${WHEEL_HREF}"
248+
wget "${WHEEL_HREF}" -O torch.whl
249+
unzip -q torch.whl -d ./_torch_whl_x
250+
mv ./_torch_whl_x/torch libtorch
251+
rm -rf ./_torch_whl_x torch.whl
252+
msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch"
253+
else
254+
msg "Downloading LibTorch ${LIBTORCH_VERSION}..."
255+
wget "${LIBTORCH_URL}" -O libtorch.zip
256+
msg "Extracting LibTorch..."
257+
unzip -q libtorch.zip
258+
rm libtorch.zip
259+
msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch"
260+
fi
227261
else
228262
msg "[SKIPPED] LibTorch already installed"
229263
fi

‎packages/feedsim/install_feedsim_aarch64.sh‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -182,11 +182,15 @@ else
182182
msg "[SKIPPED] glog-${DEP_GFLAGS_VERSION}"
183183
fi
184184

185-
DEP_JEMALLOC_VERSION="5.3.0"
185+
DEP_JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}"
186186
# Installing JEMalloc
187187
if ! [ -d "jemalloc-${DEP_JEMALLOC_VERSION}" ]; then
188188
wget "https://github.com/jemalloc/jemalloc/releases/download/${DEP_JEMALLOC_VERSION}/jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" -O "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2"
189-
verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa"
189+
if [ "${DEP_JEMALLOC_VERSION}" = "5.3.0" ]; then
190+
verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa"
191+
else
192+
msg "[WARN] no pinned checksum for jemalloc ${DEP_JEMALLOC_VERSION}; skipping verify (official github release over https)"
193+
fi
190194
bunzip2 "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2"
191195
tar -xvf "jemalloc-${DEP_JEMALLOC_VERSION}.tar"
192196
cd "jemalloc-${DEP_JEMALLOC_VERSION}"
@@ -198,11 +202,15 @@ else
198202
msg "[SKIPPED] jemalloc-${DEP_JEMALLOC_VERSION}"
199203
fi
200204

201-
DEP_LIBEVENT_VERSION="2.1.12-stable"
205+
DEP_LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}"
202206
# Installing libevent
203207
if ! [ -d "libevent-${DEP_LIBEVENT_VERSION}" ]; then
204208
wget "https://github.com/libevent/libevent/releases/download/release-${DEP_LIBEVENT_VERSION}/libevent-${DEP_LIBEVENT_VERSION}.tar.gz" -O "libevent-${DEP_LIBEVENT_VERSION}.tar.gz"
205-
verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb"
209+
if [ "${DEP_LIBEVENT_VERSION}" = "2.1.12-stable" ]; then
210+
verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb"
211+
else
212+
msg "[WARN] no pinned checksum for libevent ${DEP_LIBEVENT_VERSION}; skipping verify (official github release over https)"
213+
fi
206214
tar -xzf "libevent-${DEP_LIBEVENT_VERSION}.tar.gz"
207215
cd "libevent-${DEP_LIBEVENT_VERSION}"
208216
./configure
@@ -242,9 +250,14 @@ if ! [ -d "libtorch" ]; then
242250
export PATH="${CONDA_DIR}/bin:${PATH}"
243251

244252
# Install CPU-only PyTorch via pip — this is the only reliable way to get
245-
# CPU-only libtorch on aarch64
246-
msg "Installing PyTorch CPU-only via pip..."
247-
pip install torch --index-url https://download.pytorch.org/whl/cpu
253+
# CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version;
254+
# unset reproduces the v2 baseline (latest).
255+
msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..."
256+
if [ -n "${LIBTORCH_VERSION:-}" ]; then
257+
pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu
258+
else
259+
pip install torch --index-url https://download.pytorch.org/whl/cpu
260+
fi
248261

249262
# Also install libstdcxx-ng to ensure compatible C++ runtime
250263
eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)"

‎packages/feedsim/install_feedsim_aarch64_ubuntu.sh‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P)
2727
BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")"
2828
FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim"
2929
FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party"
30+
LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}"
3031
DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz"
3132
echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}"
3233

@@ -204,9 +205,14 @@ if ! [ -d "libtorch" ]; then
204205
export PATH="${CONDA_DIR}/bin:${PATH}"
205206

206207
# Install CPU-only PyTorch via pip — this is the only reliable way to get
207-
# CPU-only libtorch on aarch64
208-
msg "Installing PyTorch CPU-only via pip..."
209-
pip install torch --index-url https://download.pytorch.org/whl/cpu
208+
# CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version;
209+
# empty falls back to pip's latest resolution.
210+
msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..."
211+
if [ -n "${LIBTORCH_VERSION:-}" ]; then
212+
pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu
213+
else
214+
pip install torch --index-url https://download.pytorch.org/whl/cpu
215+
fi
210216

211217
# Also install libstdcxx-ng to ensure compatible C++ runtime
212218
eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)"

‎packages/feedsim/install_feedsim_ubuntu.sh‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,14 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P)
1111
BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")"
1212
FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim"
1313
FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party"
14-
LIBTORCH_VERSION="2.8.0"
14+
LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}"
15+
# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel
16+
# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip.
17+
# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no
18+
# standalone zip); harmless for older versions. Default 1 pairs with the
19+
# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without
20+
# additional env overrides.
21+
LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}"
1522
DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz"
1623
echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}"
1724

@@ -213,12 +220,31 @@ else
213220
fi
214221

215222
if ! [ -d "libtorch" ]; then
216-
msg "Downloading LibTorch ${LIBTORCH_VERSION}..."
217-
wget "${LIBTORCH_URL}" -O libtorch.zip
218-
msg "Extracting LibTorch..."
219-
unzip -q libtorch.zip
220-
rm libtorch.zip
221-
msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch"
223+
if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then
224+
# Extract LibTorch from the prebuilt torch CPU wheel. The wheel's
225+
# torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as
226+
# the standalone libtorch zip, so we just rename it to libtorch/.
227+
msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..."
228+
# The C++ libtorch inside (torch/lib, torch/share/cmake) is Python-
229+
# version independent, so the cp310 wheel is fine for our C++ link.
230+
WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \
231+
| grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \
232+
| head -1)"
233+
[ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index"
234+
msg "Wheel: ${WHEEL_HREF}"
235+
wget "${WHEEL_HREF}" -O torch.whl
236+
unzip -q torch.whl -d ./_torch_whl_x
237+
mv ./_torch_whl_x/torch libtorch
238+
rm -rf ./_torch_whl_x torch.whl
239+
msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch"
240+
else
241+
msg "Downloading LibTorch ${LIBTORCH_VERSION}..."
242+
wget "${LIBTORCH_URL}" -O libtorch.zip
243+
msg "Extracting LibTorch..."
244+
unzip -q libtorch.zip
245+
rm libtorch.zip
246+
msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch"
247+
fi
222248
else
223249
msg "[SKIPPED] LibTorch already installed"
224250
fi

0 commit comments

Comments
 (0)