Skip to content

Commit d66f306

Browse files
anloweeanlowee
committed
Merge remote-tracking branch 'xwei/main' into clp_integration_07_23
2 parents 6ff3d31 + ff41c35 commit d66f306

File tree

1,004 files changed

+70253
-19852
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,004 files changed

+70253
-19852
lines changed

‎.dockerignore‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
<<<<<<<< HEAD:.github/zizmor.yml
15+
rules:
16+
use-trusted-publishing:
17+
ignore:
18+
- build_pyvelox.yml
19+
========
20+
21+
_build/
22+
23+
# Python binary dirs
24+
.venv/
25+
dist/
26+
wheelhouse/
27+
>>>>>>>> xwei/main:.dockerignore

‎.github/CODEOWNERS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ scripts/ @assignUser @majetideepak
3232
# Breeze
3333
velox/experimental/breeze @dreveman
3434

35+
# cuDF
36+
velox/experimental/cudf/ @bdice @karthikeyann @devavret @mhaseeb123
37+
3538
# Parquet
3639
velox/dwio/parquet/ @majetideepak
3740

‎.github/workflows/benchmark.yml‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Ubuntu Benchmark
16+
17+
on:
18+
pull_request:
19+
paths:
20+
- .github/workflows/benchmark.yml
21+
- scripts/ci/benchmark-requirements.txt
22+
- scripts/setup-ubuntu.sh
23+
24+
push:
25+
branches: [main]
26+
27+
permissions:
28+
contents: read
29+
30+
concurrency:
31+
group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
32+
cancel-in-progress: true
33+
34+
defaults:
35+
run:
36+
shell: bash
37+
38+
jobs:
39+
benchmark:
40+
if: github.repository == 'facebookincubator/velox'
41+
runs-on: 8-core-ubuntu-22.04
42+
env:
43+
CCACHE_DIR: ${{ github.workspace }}/ccache/
44+
CCACHE_BASEDIR: ${{ github.workspace }}
45+
BINARY_DIR: ${{ github.workspace }}/benchmarks/
46+
CONTENDER_OUTPUT_PATH: ${{ github.workspace }}/benchmark-results/contender/
47+
INSTALL_PREFIX: ${{ github.workspace }}/dependencies
48+
steps:
49+
50+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
51+
with:
52+
path: velox
53+
54+
persist-credentials: false
55+
56+
- name: Restore Dependencies
57+
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
58+
id: restore-deps
59+
with:
60+
path: ${{ env.INSTALL_PREFIX }}
61+
key: dependencies-benchmark-${{ hashFiles('velox/scripts/setup-ubuntu.sh') }}
62+
63+
- name: Install apt dependencies
64+
run: |
65+
source velox/scripts/setup-ubuntu.sh
66+
install_apt_deps
67+
68+
- name: Install compiled dependencies
69+
if: ${{ steps.restore-deps.outputs.cache-hit != 'true' }}
70+
env:
71+
CCACHE_DISABLE: 'true'
72+
run: |
73+
source velox/scripts/setup-ubuntu.sh
74+
run_and_time install_fmt
75+
run_and_time install_protobuf
76+
run_and_time install_boost
77+
run_and_time install_fast_float
78+
run_and_time install_folly
79+
run_and_time install_stemmer
80+
run_and_time install_thrift
81+
run_and_time install_arrow
82+
83+
- name: Save Dependencies
84+
if: ${{ steps.restore-deps.outputs.cache-hit != 'true' }}
85+
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
86+
with:
87+
path: ${{ env.INSTALL_PREFIX }}
88+
key: dependencies-benchmark-${{ hashFiles('velox/scripts/setup-ubuntu.sh') }}
89+
90+
- name: Restore ccache
91+
uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
92+
id: restore-cache
93+
with:
94+
clean: true
95+
path: ccache
96+
key: ccache-benchmark
97+
98+
- name: Clear CCache Statistics
99+
run: |
100+
ccache -sz
101+
102+
- name: Build Contender Benchmarks
103+
working-directory: velox
104+
run: |
105+
n_cores=$(nproc)
106+
make benchmarks-basic-build NUM_THREADS=$n_cores MAX_HIGH_MEM_JOBS=$n_cores MAX_LINK_JOBS=$n_cores
107+
mkdir -p ${BINARY_DIR}/contender/
108+
cp -r --verbose _build/release/velox/benchmarks/basic/* ${BINARY_DIR}/contender/
109+
110+
- name: CCache after
111+
run: |
112+
ccache -vs
113+
114+
- name: Save ccache"
115+
uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
116+
with:
117+
path: ccache
118+
key: ccache-benchmark
119+
120+
- name: Install benchmark dependencies
121+
run: |
122+
python3 -m pip install -r velox/scripts/ci/benchmark-requirements.txt
123+
124+
- name: Run Benchmarks - Contender
125+
working-directory: velox
126+
run: |
127+
make benchmarks-basic-run \
128+
EXTRA_BENCHMARK_FLAGS="--binary_path ${BINARY_DIR}/contender/ --output_path ${CONTENDER_OUTPUT_PATH}"
129+
130+
- name: Upload result artifact
131+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
132+
with:
133+
path: benchmark-results
134+
name: benchmark-results
135+
retention-days: 5
136+
137+
upload:
138+
runs-on: ubuntu-latest
139+
needs: benchmark
140+
if: false && github.event_name == 'push'
141+
permissions:
142+
actions: read
143+
statuses: write
144+
steps:
145+
146+
- name: Download artifacts
147+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
148+
with:
149+
merge-multiple: true
150+
path: /tmp/artifacts/
151+
152+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
153+
with:
154+
path: velox
155+
persist-credentials: false
156+
157+
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
158+
with:
159+
python-version: '3.10'
160+
cache: pip
161+
cache-dependency-path: velox/scripts/*
162+
163+
- name: Install dependencies
164+
run: pip install -r velox/scripts/ci/benchmark-requirements.txt
165+
166+
- name: Upload results
167+
env:
168+
CONBENCH_URL: https://velox-conbench.voltrondata.run/
169+
CONBENCH_MACHINE_INFO_NAME: GitHub-runner-8-core
170+
CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
171+
CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASSWORD }}
172+
CONBENCH_PROJECT_REPOSITORY: ${{ github.repository }}
173+
CONBENCH_PROJECT_COMMIT: ${{ github.sha }}
174+
RUN_ID: GHA-${{ github.run_id }}-${{ github.run_attempt }}
175+
run: |
176+
./velox/scripts/ci/benchmark-runner.py upload \
177+
--run_id "$RUN_ID" \
178+
--sha "$CONBENCH_PROJECT_COMMIT" \
179+
--output_dir "/tmp/artifacts/contender/"
180+
181+
- name: Check the status of the upload
182+
# Status functions like failure() only work in `if:`
183+
if: failure()
184+
id: status
185+
run: echo "failed=true" >> $GITHUB_OUTPUT
186+
187+
- name: Create a GitHub Status on the contender commit (whether the upload was successful)
188+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
189+
if: ${{ !cancelled() && steps.extract.conclusion != 'failure' }}
190+
with:
191+
script: |
192+
let url = 'https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }}'
193+
let state = 'success'
194+
let description = 'Result upload succeeded!'
195+
196+
if(${{ steps.status.outputs.failed == 'true' && true || false }}) {
197+
state = 'failure'
198+
description = 'Result upload failed!'
199+
}
200+
201+
github.rest.repos.createCommitStatus({
202+
owner: context.repo.owner,
203+
repo: context.repo.repo,
204+
sha: context.sha,
205+
state: state,
206+
target_url: url,
207+
description: description,
208+
context: 'Benchmark Result Upload'
209+
})
210+
211+
- name: Create a GitHub Check benchmark report on the merged PR
212+
env:
213+
CONBENCH_URL: https://velox-conbench.voltrondata.run/
214+
GITHUB_APP_ID: ${{ secrets.GH_APP_ID }}
215+
GITHUB_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }}
216+
COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
217+
CONTENDER_SHA: ${{ github.sha }}
218+
run: |
219+
./velox/scripts/ci/benchmark-alert.py \
220+
--contender-sha "$CONTENDER_SHA" \
221+
--merge-commit-message "$COMMIT_MESSAGE" \
222+
--z-score-threshold 50

‎.github/workflows/breeze.yml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Breeze Linux Build
16+
17+
on:
18+
push:
19+
branches:
20+
- main
21+
paths:
22+
- velox/experimental/breeze/**
23+
- velox/external/perfetto/**
24+
- CMake/**
25+
- scripts/setup-ubuntu.sh
26+
- scripts/setup-helper-functions.sh
27+
- .github/workflows/breeze.yml
28+
29+
pull_request:
30+
paths:
31+
- velox/experimental/breeze/**
32+
- velox/external/perfetto/**
33+
- CMake/**
34+
- scripts/setup-ubuntu.sh
35+
- scripts/setup-helper-functions.sh
36+
- .github/workflows/breeze.yml
37+
38+
permissions:
39+
contents: read
40+
41+
concurrency:
42+
group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }}
43+
cancel-in-progress: true
44+
45+
jobs:
46+
ubuntu-debug:
47+
runs-on: ubuntu-22.04
48+
# prevent errors when forks ff their main branch
49+
if: ${{ github.repository == 'facebookincubator/velox' }}
50+
name: Ubuntu debug
51+
defaults:
52+
run:
53+
shell: bash
54+
working-directory: velox
55+
steps:
56+
57+
- uses: actions/checkout@v4
58+
with:
59+
path: velox
60+
persist-credentials: false
61+
62+
- name: Install Dependencies
63+
run: |
64+
source scripts/setup-ubuntu.sh && install_apt_deps
65+
66+
- name: Make Debug Build
67+
env:
68+
VELOX_DEPENDENCY_SOURCE: BUNDLED
69+
# OpenMP build with asan+ubsan enabled
70+
run: |
71+
cmake -S velox/experimental/breeze -B _build-breeze/debug \
72+
-DCMAKE_BUILD_TYPE=Asan \
73+
-DCMAKE_CXX_FLAGS="-fsanitize=undefined" \
74+
-DBUILD_GENERATE_TEST_FIXTURES=OFF \
75+
-DBUILD_OPENMP=ON
76+
cmake --build _build-breeze/debug -j 8
77+
78+
- name: Run Tests
79+
run: |
80+
cd _build-breeze/debug && ctest -j 8 --output-on-failure --no-tests=error
81+
82+
ubuntu-gpu-relwithdebinfo:
83+
runs-on: 4-core-ubuntu-gpu-t4
84+
# prevent errors when forks ff their main branch
85+
if: ${{ github.repository == 'facebookincubator/velox' }}
86+
name: Ubuntu GPU debug
87+
env:
88+
CUDA_VERSION: '12.2'
89+
defaults:
90+
run:
91+
shell: bash
92+
working-directory: velox
93+
steps:
94+
95+
- uses: actions/checkout@v4
96+
with:
97+
path: velox
98+
persist-credentials: false
99+
100+
- name: Install Dependencies
101+
run: |
102+
source scripts/setup-ubuntu.sh && install_apt_deps && install_cuda ${CUDA_VERSION}
103+
sudo chmod 755 -R /usr/local/lib/python3.10/dist-packages
104+
105+
- name: Make RelWithDebInfo Build
106+
run: |
107+
cmake -S velox/experimental/breeze -B _build-breeze/relwithdebinfo \
108+
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
109+
-DBUILD_GENERATE_TEST_FIXTURES=OFF \
110+
-DBUILD_CUDA=ON \
111+
-DCMAKE_NVCC_FLAGS="-arch=native"
112+
cmake --build _build-breeze/relwithdebinfo -j 8
113+
114+
- name: Run Tests
115+
run: |
116+
cd _build-breeze/relwithdebinfo && ctest -j 8 --output-on-failure --no-tests=error

0 commit comments

Comments
 (0)