Skip to content

Commit ed9b9d7

Browse files
author
Ahmad El Youssef
committed
Port CentOS 9/10 support to main
1 parent 9308c3e commit ed9b9d7

32 files changed

Lines changed: 2059 additions & 119 deletions

‎.github/scripts/dcperf_install_django.bash‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,11 @@ install_django () {
3030
echo ""
3131
fi
3232

33-
echo "[TEST] TODO: Implement This!"
33+
34+
# shellcheck disable=SC2155
35+
local env_prefix=$(env_name_or_prefix "${env_name}")
36+
37+
echo "[INSTALL] Installing Django ..."
38+
(print_exec conda run --no-capture-output ${env_prefix} \
39+
python ./benchpress_cli.py install django_workload_default) || return 1
3440
}

‎.github/scripts/dcperf_install_health_check.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install Health Check
15+
################################################################################
16+
17+
install_health_check () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install Health Check"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[INSTALL] Installing Health Check ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py install health_check) || return 1
39+
}

‎.github/scripts/dcperf_install_spark.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install Spark
15+
################################################################################
16+
17+
install_spark () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install Spark"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[INSTALL] Installing Spark ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py install spark_standalone_remote) || return 1
39+
}

‎.github/scripts/dcperf_install_tao_bench.bash‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# tao_bench Install Functions
15+
################################################################################
16+
17+
install_tao_bench () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install Tao_bench"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
34+
# shellcheck disable=SC2155
35+
local env_prefix=$(env_name_or_prefix "${env_name}")
36+
37+
echo "[INSTALL] Installing Tao_bench ..."
38+
(print_exec conda run --no-capture-output ${env_prefix} \
39+
python ./benchpress_cli.py install tao_bench_autoscale) || return 1
40+
}

‎.github/scripts/dcperf_install_video_transcode_bench.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install FeedSim
15+
################################################################################
16+
17+
install_video_transcode_bench () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install VideoTranscodeBench"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
clang --version
36+
echo "[INSTALL] Installing VideoTranscodeBench ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py install video_transcode_bench_svt) || return 1
39+
}

‎.github/scripts/dcperf_install_wdl.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Install wdl
15+
################################################################################
16+
17+
install_wdl () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Install wdl"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[INSTALL] Installing wdl ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py -b wdl install folly_single_core) || return 1
39+
}

‎.github/scripts/setup_env.bash‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,13 @@
1717
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_django.bash"
1818
# shellcheck disable=SC1091,SC2128
1919
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_feedsim.bash"
20+
# shellcheck disable=SC1091,SC2128
21+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_tao_bench.bash"
22+
# shellcheck disable=SC1091,SC2128
23+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_spark.bash"
24+
# shellcheck disable=SC1091,SC2128
25+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_video_transcode_bench.bash"
26+
# shellcheck disable=SC1091,SC2128
27+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_wdl.bash"
28+
# shellcheck disable=SC1091,SC2128
29+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_health_check.bash"

‎.github/scripts/utils_build.bash‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,38 @@ install_python_tools () {
9090
# shellcheck disable=SC2086
9191
(exec_with_retries 3 conda install ${env_prefix} -c conda-forge --override-channels -y \
9292
click \
93-
numpy \
93+
numpy=1.* \
9494
pandas \
9595
pyyaml \
96-
tabulate) || return 1
96+
tabulate \
97+
packaging) || return 1
9798

9899
# Check Python packages are importable
99-
local import_tests=( click numpy pandas tabulate )
100+
local import_tests=( click numpy pandas tabulate packaging )
100101
for p in "${import_tests[@]}"; do
101102
(test_python_import_package "${env_name}" "${p}") || return 1
102103
done
103104

104105
echo "[INSTALL] Successfully installed all the build tools"
105106
}
107+
108+
################################################################################
109+
# HHVM Setup
110+
################################################################################
111+
112+
install_hhvm (){
113+
echo "[INSTALL] Installing hhvm ..."
114+
if which dnf; then
115+
wget https://github.com/facebookresearch/DCPerf/releases/download/hhvm/hhvm-3.30-multplatform-binary.tar.xz
116+
tar -Jxf hhvm-3.30-multplatform-binary.tar.xz
117+
else
118+
wget https://github.com/facebookresearch/DCPerf/releases/download/hhvm/hhvm-3.30-multplatform-binary-ubuntu.tar.xz
119+
tar -Jxf hhvm-3.30-multplatform-binary-ubuntu.tar.xz
120+
apt-get update
121+
apt-get install -y libpcre3 libunwind8
122+
123+
fi
124+
cd hhvm || return 1
125+
bash -x ./pour-hhvm.sh
126+
cd .. || return 1
127+
}

‎.github/workflows/dcperf_ci_django.yml‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,21 @@
77
name: DCPerf Django CI
88

99
on:
10-
# PR Trigger
10+
# PR Trigger — only run when files affecting this benchmark change
1111
#
1212
pull_request:
13+
paths:
14+
- 'packages/common/os-distro.sh'
15+
- 'packages/django_workload/**'
1316

1417
# Push Trigger (enable to catch errors coming out of multiple merges)
1518
#
1619
push:
1720
branches:
1821
- main
22+
paths:
23+
- 'packages/common/os-distro.sh'
24+
- 'packages/django_workload/**'
1925

2026
# Manual Trigger
2127
#
@@ -31,7 +37,7 @@ jobs:
3137
build_artifact:
3238
runs-on: ${{ matrix.host-machine.instance }}
3339
container:
34-
image: ubuntu:22.04
40+
image: ${{ matrix.host-machine.container }}
3541
options: --user root --privileged --pid=host
3642
volumes:
3743
- /var/run/docker.sock:/var/run/docker.sock
@@ -41,18 +47,40 @@ jobs:
4147
env:
4248
PRELUDE: .github/scripts/setup_env.bash
4349
BUILD_ENV: build_env
44-
continue-on-error: true
50+
continue-on-error: false
4551
strategy:
4652
fail-fast: false
4753
matrix:
4854
host-machine: [
49-
{ arch: x86, instance: "ubuntu-latest" },
55+
{ arch: x86, instance: "ubuntu-latest", container: "quay.io/centos/centos:stream9", type: "centos" },
56+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "quay.io/centos/centos:stream9", type: "centos" },
57+
{ arch: x86, instance: "ubuntu-latest", container: "quay.io/centos/centos:stream10", type: "centos" },
58+
{ arch: aarch64, instance: "ubuntu-24.04-arm", container: "quay.io/centos/centos:stream10", type: "centos" },
5059
]
5160
python-version: [ "3.12" ]
52-
5361
steps:
54-
- name: Setup Build Container
55-
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget
62+
- name: Setup Build Container (ubuntu-based)
63+
if: ${{ matrix.host-machine.type == 'ubuntu' }}
64+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget python3-dev
65+
66+
- name: Setup Build Container (centos-based)
67+
if: ${{ matrix.host-machine.type == 'centos' }}
68+
run: dnf update -y; dnf install -y git pciutils which python3-devel sudo shadow-utils passwd ; dnf group install -y "Development Tools" --exclude="texlive*"
69+
70+
- name: Fix PAM configuration for sudo
71+
if: ${{ matrix.host-machine.type == 'centos' }}
72+
run: |
73+
echo "auth sufficient pam_rootok.so" > /etc/pam.d/sudo
74+
75+
- name: Enable Sudo without password
76+
if: ${{ matrix.host-machine.type == 'centos' }}
77+
run: |
78+
echo "root ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers
79+
80+
- name: Unlock root user account
81+
if: ${{ matrix.host-machine.type == 'centos' }}
82+
run: |
83+
passwd -u root
5684
5785
- name: Checkout the Repository
5886
uses: actions/checkout@v4
@@ -65,14 +93,17 @@ jobs:
6593
- name: Display System Info
6694
run: . $PRELUDE; print_system_info
6795

96+
- name: Install Build Tools
97+
run: . $PRELUDE; install_build_tools
98+
6899
- name: Setup Miniconda
69100
run: . $PRELUDE; setup_miniconda $HOME/miniconda
70101

71102
- name: Create Conda Environment
72103
run: . $PRELUDE; create_conda_environment $BUILD_ENV ${{ matrix.python-version }}
73104

74105
- name: Install Python Tools
75-
run: . $PRELUDE; install_build_tools $BUILD_ENV
106+
run: . $PRELUDE; install_python_tools $BUILD_ENV
76107

77108
- name: Install Django
78109
run: . $PRELUDE; install_django $BUILD_ENV

0 commit comments

Comments
 (0)