Skip to content

Commit ba5da13

Browse files
assignUserfacebook-github-bot
assignUser
authored and
facebook-github-bot
committed
refactor(ci): Cleanup scripts/ (facebookincubator#13345)
Summary: The `scripts/` dir has become a bit unwieldy. This is a first pass moving the docker and CI related files. I will continue with the check related files when adding pre-commit. See also facebookincubator#13028 Pull Request resolved: facebookincubator#13345 Reviewed By: tanjialiang Differential Revision: D75028015 Pulled By: xiaoxmeng fbshipit-source-id: 94d54643a10c3dc704ecd4bbc666422d67a93fb4
1 parent 23f7017 commit ba5da13

31 files changed

+61
-243
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ on:
1818
pull_request:
1919
paths:
2020
- '.github/workflows/benchmark.yml'
21-
- 'scripts/benchmark-requirements.txt'
22-
- 'scripts/setup-ubuntu.sh'
21+
- 'scripts/ci/benchmark-requirements.txt'
22+
- 'scripts/ci/setup-ubuntu.sh'
2323

2424
push:
2525
branches: [main]
@@ -126,7 +126,7 @@ jobs:
126126

127127
- name: Install benchmark dependencies
128128
run: |
129-
python3 -m pip install -r velox/scripts/benchmark-requirements.txt
129+
python3 -m pip install -r velox/scripts/ci/benchmark-requirements.txt
130130
131131
- name: Run Benchmarks - Contender
132132
working-directory: 'velox'
@@ -168,7 +168,7 @@ jobs:
168168
cache-dependency-path: "velox/scripts/*"
169169

170170
- name: Install dependencies
171-
run: pip install -r velox/scripts/benchmark-requirements.txt
171+
run: pip install -r velox/scripts/ci/benchmark-requirements.txt
172172

173173
- name: Upload results
174174
env:
@@ -180,7 +180,7 @@ jobs:
180180
CONBENCH_PROJECT_COMMIT: "${{ github.sha }}"
181181
RUN_ID: "GHA-${{ github.run_id }}-${{ github.run_attempt }}"
182182
run: |
183-
./velox/scripts/benchmark-runner.py upload \
183+
./velox/scripts/ci/benchmark-runner.py upload \
184184
--run_id "$RUN_ID" \
185185
--sha "$CONBENCH_PROJECT_COMMIT" \
186186
--output_dir "/tmp/artifacts/contender/"
@@ -223,7 +223,7 @@ jobs:
223223
COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
224224
CONTENDER_SHA: "${{ github.sha }}"
225225
run: |
226-
./velox/scripts/benchmark-alert.py \
226+
./velox/scripts/ci/benchmark-alert.py \
227227
--contender-sha "$CONTENDER_SHA" \
228228
--merge-commit-message "$COMMIT_MESSAGE" \
229229
--z-score-threshold 50

‎.github/workflows/build-metrics.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ on:
1818
pull_request:
1919
paths:
2020
- ".github/workflows/build-metrics.yml"
21-
- "scripts/bm-report/**"
21+
- "scripts/ci/bm-report/**"
2222

2323
workflow_dispatch:
2424
inputs:
@@ -111,7 +111,7 @@ jobs:
111111
- name: "Install dependencies"
112112
run: |
113113
python3 -m pip install setuptools
114-
python3 -m pip install -r scripts/benchmark-requirements.txt
114+
python3 -m pip install -r scripts/ci/benchmark-requirements.txt
115115
116116
- name: "Upload Metrics"
117117
# This disables the upload and report generation on fork PRs but allows it for forks from within the main repo.
@@ -126,7 +126,7 @@ jobs:
126126
CONBENCH_PROJECT_REPOSITORY: "${{ github.repository }}"
127127
CONBENCH_PROJECT_COMMIT: "${{ inputs.ref || github.sha }}"
128128
run: |
129-
./scripts/build-metrics.py upload \
129+
./scripts/ci/bm-report/build-metrics.py upload \
130130
--build_type "${{ matrix.link-type }}-${{ matrix.type }}" \
131131
--run_id "BM-${{ matrix.link-type }}-${{ matrix.type }}-${{ github.run_id }}-${{ github.run_attempt }}" \
132132
--pr_number "${{ github.event.number }}" \
@@ -156,7 +156,7 @@ jobs:
156156

157157
- name: Build Environment
158158
run: |
159-
cd scripts/bm-report
159+
cd scripts/ci/bm-report
160160
nix-build
161161
162162
- name: Build Documentation
@@ -165,14 +165,14 @@ jobs:
165165
CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}"
166166
CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}"
167167
run: |
168-
cd scripts/bm-report
168+
cd scripts/ci/bm-report
169169
nix-shell --run "quarto render report.qmd"
170170
171171
- name: Upload Report Artifact
172172
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
173173
with:
174174
name: report
175-
path: scripts/bm-report/report.html
175+
path: scripts/ci/bm-report/report.html
176176
retention-days: 5
177177

178178
- name: Push Report
@@ -182,7 +182,7 @@ jobs:
182182
run: |
183183
git checkout gh-pages
184184
mkdir -p docs/bm-report
185-
cp -R scripts/bm-report/report.html docs/bm-report/index.html
185+
cp -R scripts/ci/bm-report/report.html docs/bm-report/index.html
186186
git add docs
187187
188188
if [ -n "$(git status --porcelain --untracked-files=no)" ]

‎.github/workflows/docker.yml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,16 @@ name: Build & Push Docker Images
1616
on:
1717
pull_request:
1818
paths:
19-
- scripts/*.dockfile
20-
- scripts/*.dockerfile
19+
- scripts/docker/*.dockfile
20+
- scripts/docker/*.dockerfile
2121
- scripts/setup-*.sh
2222
- .github/workflows/docker.yml
2323
- docker-compose.yml
2424
push:
2525
branches: [main]
2626
paths:
27-
- scripts/*.dockfile
28-
- scripts/*.dockerfile
27+
- scripts/docker/*.dockfile
28+
- scripts/docker/*.dockerfile
2929
- scripts/setup-*.sh
3030
- .github/workflows/docker.yml
3131

@@ -46,16 +46,16 @@ jobs:
4646
matrix:
4747
include:
4848
- name: Check
49-
file: "scripts/check-container.dockfile"
49+
file: "scripts/docker/check-container.dockfile"
5050
tags: "ghcr.io/facebookincubator/velox-dev:check"
5151
- name: Centos 9
52-
file: "scripts/centos.dockerfile"
52+
file: "scripts/docker/centos.dockerfile"
5353
tags: "ghcr.io/facebookincubator/velox-dev:centos9"
5454
- name: Pyvelox
55-
file: "scripts/pyvelox.dockerfile"
55+
file: "scripts/docker/pyvelox.dockerfile"
5656
tags: "ghcr.io/facebookincubator/velox-dev:pyvelox"
5757
- name: Dev
58-
file: "scripts/ubuntu-22.04-cpp.dockerfile"
58+
file: "scripts/docker/ubuntu-22.04-cpp.dockerfile"
5959
args: ""
6060
tags: "ghcr.io/facebookincubator/velox-dev:ubuntu-22.04"
6161

@@ -90,14 +90,14 @@ jobs:
9090
matrix:
9191
include:
9292
- name: Adapters
93-
file: "scripts/adapters.dockerfile"
93+
file: "scripts/docker/adapters.dockerfile"
9494
tags: "ghcr.io/facebookincubator/velox-dev:adapters"
9595
- name: Presto Java
96-
file: "scripts/prestojava-container.dockerfile"
96+
file: "scripts/docker/prestojava-container.dockerfile"
9797
args: "PRESTO_VERSION=0.290"
9898
tags: "ghcr.io/facebookincubator/velox-dev:presto-java"
9999
- name: Spark server
100-
file: "scripts/spark-container.dockerfile"
100+
file: "scripts/docker/spark-container.dockerfile"
101101
args: "SPARK_VERSION=3.5.1"
102102
tags: "ghcr.io/facebookincubator/velox-dev:spark-server"
103103

‎.github/workflows/scheduled.yml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -175,9 +175,9 @@ jobs:
175175
run: |
176176
source .venv/bin/activate
177177
python3 -m pip install deepdiff
178-
python3 scripts/signature.py export --spark /tmp/signatures/spark_signatures_main.json
179-
python3 scripts/signature.py export --presto /tmp/signatures/presto_signatures_main.json
180-
python3 scripts/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_main.json
178+
python3 scripts/ci/signature.py export --spark /tmp/signatures/spark_signatures_main.json
179+
python3 scripts/ci/signature.py export --presto /tmp/signatures/presto_signatures_main.json
180+
python3 scripts/ci/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_main.json
181181
182182
- name: Save Function Signature Stash
183183
if: ${{ github.event_name == 'pull_request' && steps.get-sig.outputs.stash-hit != 'true' }}
@@ -234,9 +234,9 @@ jobs:
234234
run: |
235235
source .venv/bin/activate
236236
python3 -m pip install deepdiff
237-
python3 scripts/signature.py gh_bias_check presto spark
238-
python3 scripts/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_contender.json
239-
python3 scripts/signature.py bias_aggregates /tmp/signatures/presto_aggregate_signatures_main.json \
237+
python3 scripts/ci/signature.py gh_bias_check presto spark
238+
python3 scripts/ci/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_contender.json
239+
python3 scripts/ci/signature.py bias_aggregates /tmp/signatures/presto_aggregate_signatures_main.json \
240240
/tmp/signatures/presto_aggregate_signatures_contender.json /tmp/signatures/presto_aggregate_bias_functions \
241241
/tmp/signatures/presto_aggregate_errors
242242
@@ -648,7 +648,7 @@ jobs:
648648
- name: Run Join Fuzzer
649649
run: |
650650
cd velox
651-
cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
651+
cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
652652
ls -lR $PRESTO_HOME/etc
653653
$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
654654
# Sleep for 60 seconds to allow Presto server to start.
@@ -749,7 +749,7 @@ jobs:
749749
- name: Run RowNumber Fuzzer
750750
run: |
751751
cd velox
752-
cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
752+
cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
753753
ls -lR $PRESTO_HOME/etc
754754
$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
755755
# Sleep for 60 seconds to allow Presto server to start.
@@ -811,7 +811,7 @@ jobs:
811811
- name: Run TopNRowNumber Fuzzer
812812
run: |
813813
cd velox
814-
cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
814+
cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
815815
ls -lR $PRESTO_HOME/etc
816816
$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
817817
# Sleep for 60 seconds to allow Presto server to start.
@@ -978,7 +978,7 @@ jobs:
978978
- name: "Run Aggregate Fuzzer"
979979
run: |
980980
cd velox
981-
cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
981+
cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
982982
ls -lR $PRESTO_HOME/etc
983983
$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
984984
# Sleep for 60 seconds to allow Presto server to start.
@@ -1049,7 +1049,7 @@ jobs:
10491049
- name: Run Bias Expression Fuzzer
10501050
run: |
10511051
cd velox
1052-
cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
1052+
cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
10531053
ls -lR $PRESTO_HOME/etc
10541054
$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
10551055
# Sleep for 60 seconds to allow Presto server to start.
@@ -1138,7 +1138,7 @@ jobs:
11381138
- name: "Run Bias Aggregate Fuzzer"
11391139
run: |
11401140
cd velox
1141-
cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
1141+
cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
11421142
ls -lR $PRESTO_HOME/etc
11431143
$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
11441144
# Sleep for 60 seconds to allow Presto server to start.
@@ -1231,7 +1231,7 @@ jobs:
12311231
- name: "Run Window Fuzzer"
12321232
run: |
12331233
cd velox
1234-
cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
1234+
cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
12351235
ls -lR $PRESTO_HOME/etc
12361236
$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
12371237
# Sleep for 60 seconds to allow Presto server to start.
@@ -1295,7 +1295,7 @@ jobs:
12951295
- name: "Run Writer Fuzzer"
12961296
run: |
12971297
cd velox
1298-
cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
1298+
cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
12991299
ls -lR $PRESTO_HOME/etc
13001300
echo "jvm config content:"
13011301
cat $PRESTO_HOME/etc/jvm.config

‎docker-compose.yml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,14 @@ services:
2424
image: ghcr.io/facebookincubator/velox-dev:ubuntu-22.04
2525
build:
2626
context: .
27-
dockerfile: scripts/ubuntu-22.04-cpp.dockerfile
27+
dockerfile: scripts/docker/ubuntu-22.04-cpp.dockerfile
2828
environment:
2929
NUM_THREADS: 8 # default value for NUM_THREADS
3030
VELOX_DEPENDENCY_SOURCE: BUNDLED # Build dependencies from source
3131
CCACHE_DIR: "/velox/.ccache"
3232
volumes:
3333
- .:/velox:delegated
34-
command: scripts/docker-command.sh
34+
command: scripts/docker/docker-command.sh
3535

3636
adapters-cpp:
3737
# Usage:
@@ -41,12 +41,12 @@ services:
4141
# or
4242
# docker-compose run -e NUM_THREADS=<NUMBER_OF_THREADS_TO_USE> --rm adapters-cpp
4343
# to set the number of threads used during compilation
44-
# scripts/adapters.dockerfile uses SHELL which is not supported for OCI image format.
44+
# scripts/docker/adapters.dockerfile uses SHELL which is not supported for OCI image format.
4545
# podman users must specify "--podman-build-args='--format docker'" argument.
4646
image: ghcr.io/facebookincubator/velox-dev:adapters
4747
build:
4848
context: .
49-
dockerfile: scripts/adapters.dockerfile
49+
dockerfile: scripts/docker/adapters.dockerfile
5050
args:
5151
image: ghcr.io/facebookincubator/velox-dev:centos9
5252
environment:
@@ -57,7 +57,7 @@ services:
5757
volumes:
5858
- .:/velox:delegated
5959
working_dir: /velox
60-
command: /velox/scripts/docker-command.sh
60+
command: /velox/scripts/docker/docker-command.sh
6161

6262
adapters-cuda:
6363
# Usage:
@@ -67,12 +67,12 @@ services:
6767
# or
6868
# docker-compose run -e NUM_THREADS=<NUMBER_OF_THREADS_TO_USE> --rm adapters-cuda
6969
# to set the number of threads used during compilation
70-
# scripts/adapters.dockerfile uses SHELL which is not supported for OCI image format.
70+
# scripts/docker/adapters.dockerfile uses SHELL which is not supported for OCI image format.
7171
# podman users must specify "--podman-build-args='--format docker'" argument.
7272
image: ghcr.io/facebookincubator/velox-dev:adapters
7373
build:
7474
context: .
75-
dockerfile: scripts/adapters.dockerfile
75+
dockerfile: scripts/docker/adapters.dockerfile
7676
args:
7777
image: ghcr.io/facebookincubator/velox-dev:centos9
7878
environment:
@@ -91,7 +91,7 @@ services:
9191
volumes:
9292
- .:/velox:delegated
9393
working_dir: /velox
94-
command: /velox/scripts/docker-command.sh
94+
command: /velox/scripts/docker/docker-command.sh
9595

9696

9797
centos-cpp:
@@ -104,7 +104,7 @@ services:
104104
image: ghcr.io/facebookincubator/velox-dev:centos9
105105
build:
106106
context: .
107-
dockerfile: scripts/centos.dockerfile
107+
dockerfile: scripts/docker/centos.dockerfile
108108
args:
109109
image: quay.io/centos/centos:stream9
110110
environment:
@@ -113,7 +113,7 @@ services:
113113
volumes:
114114
- .:/velox:delegated
115115
working_dir: /velox
116-
command: /velox/scripts/docker-command.sh
116+
command: /velox/scripts/docker/docker-command.sh
117117

118118
presto-java:
119119
# Usage:
@@ -127,14 +127,14 @@ services:
127127
args:
128128
- PRESTO_VERSION=0.290
129129
context: .
130-
dockerfile: scripts/prestojava-container.dockerfile
130+
dockerfile: scripts/docker/prestojava-container.dockerfile
131131
environment:
132132
NUM_THREADS: 8 # default value for NUM_THREADS
133133
CCACHE_DIR: "/velox/.ccache"
134134
volumes:
135135
- .:/velox:delegated
136136
working_dir: /velox
137-
command: /velox/scripts/docker-command.sh
137+
command: /velox/scripts/docker/docker-command.sh
138138

139139
spark-server:
140140
# Usage:
@@ -148,11 +148,11 @@ services:
148148
args:
149149
- SPARK_VERSION=3.5.1
150150
context: .
151-
dockerfile: scripts/spark-container.dockerfile
151+
dockerfile: scripts/docker/spark-container.dockerfile
152152
environment:
153153
NUM_THREADS: 8 # default value for NUM_THREADS
154154
CCACHE_DIR: "/velox/.ccache"
155155
volumes:
156156
- .:/velox:delegated
157157
working_dir: /velox
158-
command: /velox/scripts/docker-command.sh
158+
command: /velox/scripts/docker/docker-command.sh

0 commit comments

Comments
 (0)