Skip to content

Commit 5cf9f01

Browse files
wanglinsongwanglinsong
committed
Merge branch 'ci-fix-pr' into staging-rebase-pr
Signed-off-by: Linsong Wang <linsong.wang@ibm.com>
2 parents 6cba75a + 6b6532f commit 5cf9f01

File tree

1 file changed

+169
-0
lines changed

1 file changed

+169
-0
lines changed

‎.github/workflows/gluten.yml‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
# Copyright (c) Facebook, Inc. and its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: Gluten Build
16+
17+
on:
18+
pull_request:
19+
paths:
20+
- .github/workflows/gluten.yml
21+
env:
22+
MVN_CMD: ./build/mvn -ntp
23+
24+
jobs:
25+
26+
gluten-cpp-build:
27+
name: gluten cpp build
28+
# prevent errors when forks ff their main branch
29+
if: ${{ github.repository == 'IBM/velox' }}
30+
runs-on: ubuntu-22.04
31+
env:
32+
CCACHE_DIR: "${{ github.workspace }}/.ccache"
33+
steps:
34+
- uses: actions/checkout@v4
35+
- name: Get Ccache
36+
uses: actions/cache/restore@v4
37+
with:
38+
path: '${{ env.CCACHE_DIR }}'
39+
key: ccache-ibm-velox-${{github.sha}}
40+
restore-keys: |
41+
ccache-ibm-velox-
42+
ccache-ibm-velox
43+
44+
- name: Setup Gluten
45+
run: |
46+
git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
47+
sed -i 's/ibm/ibm-xxx/gI' ep/build-velox/src/get-velox.sh
48+
BRANCH=$(echo ${GITHUB_REF#refs/heads/})
49+
# sed -i 's/VELOX_BRANCH=2025.*/VELOX_BRANCH=${BRANCH}/g' ep/build-velox/src/get-velox.sh
50+
- name: Build Gluten native libraries
51+
run: |
52+
docker pull apache/gluten:vcpkg-centos-7
53+
docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c "
54+
git config --global --add safe.directory /work
55+
set -e
56+
df -a
57+
cd /work
58+
git log -n 3
59+
cd /work/gluten
60+
git log -n 3
61+
export CCACHE_DIR=/work/.ccache
62+
export CCACHE_SLOPPINESS=file_macro,locale,time_macros
63+
mkdir -p /work/.ccache
64+
ccache -M 1G
65+
ccache -sz
66+
source /opt/rh/devtoolset-11/enable
67+
source /opt/rh/rh-git227/enable
68+
export NUM_THREADS=4
69+
./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \
70+
--build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON --enable_enhanced_features=ON --velox_home=/work
71+
pushd /work
72+
git log -n 3
73+
popd
74+
ccache -s
75+
mkdir -p /work/.m2/repository/org/apache/arrow/
76+
cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
77+
"
78+
- name: "Save ccache"
79+
if: always()
80+
uses: actions/cache/save@v4
81+
id: ccache
82+
with:
83+
path: '${{ env.CCACHE_DIR }}'
84+
key: ccache-ibm-velox-${{github.sha}}
85+
86+
- uses: actions/upload-artifact@v4
87+
with:
88+
name: velox-native-lib-centos-7-${{github.sha}}
89+
path: ./gluten/cpp/build/releases/
90+
if-no-files-found: error
91+
- uses: actions/upload-artifact@v4
92+
with:
93+
name: arrow-jars-centos-7-${{github.sha}}
94+
path: .m2/repository/org/apache/arrow/
95+
if-no-files-found: error
96+
97+
spark-test-spark35:
98+
needs: gluten-cpp-build
99+
runs-on: ubuntu-22.04
100+
container: apache/gluten:centos-8-jdk8
101+
steps:
102+
- name: Setup Gluten
103+
run: |
104+
git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
105+
- name: Download All Artifacts
106+
uses: actions/download-artifact@v4
107+
with:
108+
name: velox-native-lib-centos-7-${{github.sha}}
109+
path: ./gluten/cpp/build/releases
110+
- name: Download Arrow Jars
111+
uses: actions/download-artifact@v4
112+
with:
113+
name: arrow-jars-centos-7-${{github.sha}}
114+
path: /root/.m2/repository/org/apache/arrow/
115+
- name: Build package for Spark 3.5
116+
run: |
117+
cd $GITHUB_WORKSPACE/gluten
118+
export SPARK_SCALA_VERSION=2.12
119+
yum install -y java-17-openjdk-devel
120+
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
121+
export PATH=$JAVA_HOME/bin:$PATH
122+
java -version
123+
$MVN_CMD clean package -Pspark-3.5 -Pbackends-velox -Piceberg -Pdelta -Phudi -DskipTests
124+
125+
spark-test-spark34-slow:
126+
needs: gluten-cpp-build
127+
runs-on: ubuntu-22.04
128+
container: apache/gluten:centos-8-jdk8
129+
steps:
130+
- name: Setup Gluten
131+
run: |
132+
git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten
133+
- name: Download All Artifacts
134+
uses: actions/download-artifact@v4
135+
with:
136+
name: velox-native-lib-centos-7-${{github.sha}}
137+
path: ./gluten/cpp/build/releases
138+
- name: Download Arrow Jars
139+
uses: actions/download-artifact@v4
140+
with:
141+
name: arrow-jars-centos-7-${{github.sha}}
142+
path: /root/.m2/repository/org/apache/arrow/
143+
- name: Build and Run unit test for Spark 3.4.4 (slow tests)
144+
run: |
145+
cd $GITHUB_WORKSPACE/gluten
146+
yum install -y java-17-openjdk-devel
147+
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
148+
export PATH=$JAVA_HOME/bin:$PATH
149+
java -version
150+
export SPARK_HOME=/opt/shims/spark34/spark_home/
151+
ls -l $SPARK_HOME
152+
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut -Phudi \
153+
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \
154+
-DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}"
155+
- name: Upload test report
156+
if: always()
157+
uses: actions/upload-artifact@v4
158+
with:
159+
name: ${{ github.job }}-report
160+
path: '**/surefire-reports/TEST-*.xml'
161+
- name: Upload unit tests log files
162+
if: ${{ !success() }}
163+
uses: actions/upload-artifact@v4
164+
with:
165+
name: ${{ github.job }}-test-log
166+
path: |
167+
**/target/*.log
168+
**/gluten-ut/**/hs_err_*.log
169+
**/gluten-ut/**/core.*

0 commit comments

Comments
 (0)