refactor: Extract helper method verifyOutputNames in PlanConsistencyChecker (#15370) #603
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) Facebook, Inc. and its affiliates. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: Gluten Build | |
| on: | |
| pull_request: | |
| paths: | |
| - .github/workflows/gluten.yml | |
| env: | |
| MVN_CMD: mvn -ntp | |
| jobs: | |
| gluten-cpp-build: | |
| name: gluten cpp build | |
| # prevent errors when forks ff their main branch | |
| if: ${{ github.repository == 'IBM/velox' }} | |
| runs-on: ubuntu-22.04 | |
| env: | |
| CCACHE_DIR: "${{ github.workspace }}/.ccache" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get Ccache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: '${{ env.CCACHE_DIR }}' | |
| key: ccache-centos7-release-default-${{github.sha}} | |
| restore-keys: | | |
| ccache-centos7-release-default | |
| - name: Setup Gluten | |
| run: | | |
| git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten | |
| BRANCH=$(echo ${GITHUB_REF#refs/heads/}) | |
| sed -i 's/oap-project/IBM/g' ep/build-velox/src/get-velox.sh | |
| # sed -i 's/VELOX_BRANCH=2025.*/VELOX_BRANCH=${BRANCH}/g' ep/build-velox/src/get-velox.sh | |
| - name: Build Gluten native libraries | |
| run: | | |
| docker pull apache/gluten:vcpkg-centos-7 | |
| docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c " | |
| git config --global --add safe.directory /work | |
| set -e | |
| df -a | |
| cd /work/gluten | |
| export CCACHE_DIR=/work/.ccache | |
| export CCACHE_SLOPPINESS=file_macro,locale,time_macros | |
| mkdir -p /work/.ccache | |
| ccache -sz | |
| source /opt/rh/devtoolset-11/enable | |
| source /opt/rh/rh-git227/enable | |
| export NUM_THREADS=4 | |
| ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ | |
| --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON --velox_home=/work | |
| ccache -s | |
| mkdir -p /work/.m2/repository/org/apache/arrow/ | |
| cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/ | |
| " | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: velox-native-lib-centos-7-${{github.sha}} | |
| path: ./gluten/cpp/build/releases/ | |
| if-no-files-found: error | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: arrow-jars-centos-7-${{github.sha}} | |
| path: .m2/repository/org/apache/arrow/ | |
| if-no-files-found: error | |
| spark-test-spark32: | |
| needs: gluten-cpp-build | |
| runs-on: ubuntu-22.04 | |
| container: apache/gluten:centos-8-jdk8 | |
| steps: | |
| - name: Setup Gluten | |
| run: | | |
| git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten | |
| - name: Download All Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: velox-native-lib-centos-7-${{github.sha}} | |
| path: ./gluten/cpp/build/releases | |
| - name: Download Arrow Jars | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: arrow-jars-centos-7-${{github.sha}} | |
| path: /root/.m2/repository/org/apache/arrow/ | |
| - name: Build package for Spark 3.2 | |
| run: | | |
| cd $GITHUB_WORKSPACE/gluten | |
| export SPARK_SCALA_VERSION=2.12 | |
| yum install -y java-17-openjdk-devel | |
| export JAVA_HOME=/usr/lib/jvm/java-17-openjdk | |
| export PATH=$JAVA_HOME/bin:$PATH | |
| java -version | |
| $MVN_CMD clean package -Pspark-3.2 -Pbackends-velox -Piceberg -Pdelta -Phudi -DskipTests | |
| spark-test-spark34: | |
| needs: gluten-cpp-build | |
| runs-on: ubuntu-22.04 | |
| container: apache/gluten:centos-8-jdk8 | |
| steps: | |
| - name: Setup Gluten | |
| run: | | |
| git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten | |
| - name: Download All Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: velox-native-lib-centos-7-${{github.sha}} | |
| path: ./gluten/cpp/build/releases | |
| - name: Download Arrow Jars | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: arrow-jars-centos-7-${{github.sha}} | |
| path: /root/.m2/repository/org/apache/arrow/ | |
| - name: Prepare spark.test.home for Spark 3.4.4 (other tests) | |
| run: | | |
| dnf module -y install python39 && \ | |
| alternatives --set python3 /usr/bin/python3.9 && \ | |
| pip3 install setuptools==77.0.3 && \ | |
| pip3 install pyspark==3.4.4 cython && \ | |
| pip3 install pandas==2.2.3 pyarrow==20.0.0 | |
| - name: Build and Run unit test for Spark 3.4.4 (other tests) | |
| run: | | |
| cd $GITHUB_WORKSPACE/gluten | |
| export SPARK_SCALA_VERSION=2.12 | |
| yum install -y java-17-openjdk-devel | |
| export JAVA_HOME=/usr/lib/jvm/java-17-openjdk | |
| export PATH=$JAVA_HOME/bin:$PATH | |
| java -version | |
| export SPARK_HOME=/opt/shims/spark34/spark_home/ | |
| ls -l $SPARK_HOME | |
| $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Pspark-ut \ | |
| -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ | |
| -DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}" | |
| - name: Upload test report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-report | |
| path: '**/surefire-reports/TEST-*.xml' | |
| - name: Upload unit tests log files | |
| if: ${{ !success() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-test-log | |
| path: | | |
| **/target/*.log | |
| **/gluten-ut/**/hs_err_*.log | |
| **/gluten-ut/**/core.* | |
| - name: Upload golden files | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-golden-files | |
| path: /tmp/tpch-approved-plan/** | |
| spark-test-spark34-slow: | |
| needs: gluten-cpp-build | |
| runs-on: ubuntu-22.04 | |
| container: apache/gluten:centos-8-jdk8 | |
| steps: | |
| - name: Setup Gluten | |
| run: | | |
| git clone --depth 1 https://github.com/apache/incubator-gluten gluten && cd gluten | |
| - name: Download All Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: velox-native-lib-centos-7-${{github.sha}} | |
| path: ./gluten/cpp/build/releases | |
| - name: Download Arrow Jars | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: arrow-jars-centos-7-${{github.sha}} | |
| path: /root/.m2/repository/org/apache/arrow/ | |
| - name: Build and Run unit test for Spark 3.4.4 (slow tests) | |
| run: | | |
| cd $GITHUB_WORKSPACE/gluten | |
| yum install -y java-17-openjdk-devel | |
| export JAVA_HOME=/usr/lib/jvm/java-17-openjdk | |
| export PATH=$JAVA_HOME/bin:$PATH | |
| java -version | |
| export SPARK_HOME=/opt/shims/spark34/spark_home/ | |
| ls -l $SPARK_HOME | |
| $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Pspark-ut -Phudi \ | |
| -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ | |
| -DargLine="-Dspark.test.home=$SPARK_HOME ${EXTRA_FLAGS}" | |
| - name: Upload test report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-report | |
| path: '**/surefire-reports/TEST-*.xml' | |
| - name: Upload unit tests log files | |
| if: ${{ !success() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-test-log | |
| path: | | |
| **/target/*.log | |
| **/gluten-ut/**/hs_err_*.log | |
| **/gluten-ut/**/core.* |