datafusion-comet/.github/workflows/spark_sql_test_4_0.yml at 8b5f1729844c71f8a0c9da0406cac2440a59c4ce · apache/datafusion-comet · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Runs on every PR and on main. Spark 4.0 is the newest stable Spark line.
name: Spark SQL Tests (Spark 4.0)

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

on:
  push:
    branches:
      - main
    paths:
      - "native/**/src/**"
      - "native/**/Cargo.toml"
      - "native/Cargo.lock"
      - "!native/hdfs/**"
      - "!native/fs-hdfs/**"
      - "common/src/main/**"
      - "common/pom.xml"
      - "spark/src/main/**"
      - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
      - "spark/pom.xml"
      - "dev/diffs/**"
      - "pom.xml"
      - "rust-toolchain.toml"
      - ".github/workflows/spark_sql_test_4_0.yml"
      - ".github/workflows/spark_sql_test_reusable.yml"
      - ".github/actions/setup-builder/**"
      - ".github/actions/setup-spark-builder/**"
  pull_request:
    paths:
      - "native/**/src/**"
      - "native/**/Cargo.toml"
      - "native/Cargo.lock"
      - "!native/hdfs/**"
      - "!native/fs-hdfs/**"
      - "common/src/main/**"
      - "common/pom.xml"
      - "spark/src/main/**"
      - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
      - "spark/pom.xml"
      - "dev/diffs/**"
      - "pom.xml"
      - "rust-toolchain.toml"
      - ".github/workflows/spark_sql_test_4_0.yml"
      - ".github/workflows/spark_sql_test_reusable.yml"
      - ".github/actions/setup-builder/**"
      - ".github/actions/setup-spark-builder/**"
  workflow_dispatch:
    inputs:
      collect-fallback-logs:
        description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
        required: false
        default: false
        type: boolean

jobs:
  spark-sql:
    uses: ./.github/workflows/spark_sql_test_reusable.yml
    with:
      spark-short: '4.0'
      spark-full: '4.0.2'
      java: 21
      collect-fallback-logs: ${{ github.event.inputs.collect-fallback-logs == 'true' }}