Skip to content

Commit b2e6d03

Browse files
andygroveandygrove
authored
Merge branch 'main' into fix_comet_sum_compatbility_level
2 parents c4e06ec + 2da24da commit b2e6d03

15 files changed

Lines changed: 896 additions & 256 deletions

File tree

‎.github/workflows/iceberg_spark_test.yml‎

Lines changed: 34 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,27 +25,41 @@ on:
2525
push:
2626
branches:
2727
- main
28-
paths-ignore:
29-
- "benchmarks/**"
30-
- "doc/**"
31-
- "docs/**"
32-
- "**.md"
33-
- "native/core/benches/**"
34-
- "native/spark-expr/benches/**"
35-
- "spark/src/test/**"
36-
- "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
37-
- "spark-integration/**"
28+
paths:
29+
- "native/**/src/**"
30+
- "native/**/Cargo.toml"
31+
- "native/Cargo.lock"
32+
- "!native/hdfs/**"
33+
- "!native/fs-hdfs/**"
34+
- "common/src/main/**"
35+
- "common/pom.xml"
36+
- "spark/src/main/**"
37+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
38+
- "spark/pom.xml"
39+
- "dev/diffs/iceberg/**"
40+
- "pom.xml"
41+
- "rust-toolchain.toml"
42+
- ".github/workflows/iceberg_spark_test.yml"
43+
- ".github/actions/setup-builder/**"
44+
- ".github/actions/setup-iceberg-builder/**"
3845
pull_request:
39-
paths-ignore:
40-
- "benchmarks/**"
41-
- "doc/**"
42-
- "docs/**"
43-
- "**.md"
44-
- "native/core/benches/**"
45-
- "native/spark-expr/benches/**"
46-
- "spark/src/test/**"
47-
- "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
48-
- "spark-integration/**"
46+
paths:
47+
- "native/**/src/**"
48+
- "native/**/Cargo.toml"
49+
- "native/Cargo.lock"
50+
- "!native/hdfs/**"
51+
- "!native/fs-hdfs/**"
52+
- "common/src/main/**"
53+
- "common/pom.xml"
54+
- "spark/src/main/**"
55+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
56+
- "spark/pom.xml"
57+
- "dev/diffs/iceberg/**"
58+
- "pom.xml"
59+
- "rust-toolchain.toml"
60+
- ".github/workflows/iceberg_spark_test.yml"
61+
- ".github/actions/setup-builder/**"
62+
- ".github/actions/setup-iceberg-builder/**"
4963
# manual trigger
5064
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
5165
workflow_dispatch:

‎.github/workflows/spark_sql_test_3_4.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on main only. Extra coverage for the oldest supported Spark.
19+
name: Spark SQL Tests (Spark 3.4)
20+
21+
concurrency:
22+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23+
cancel-in-progress: true
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
paths:
30+
- "native/**/src/**"
31+
- "native/**/Cargo.toml"
32+
- "native/Cargo.lock"
33+
- "!native/hdfs/**"
34+
- "!native/fs-hdfs/**"
35+
- "common/src/main/**"
36+
- "common/pom.xml"
37+
- "spark/src/main/**"
38+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
39+
- "spark/pom.xml"
40+
- "dev/diffs/**"
41+
- "pom.xml"
42+
- "rust-toolchain.toml"
43+
- ".github/workflows/spark_sql_test_3_4.yml"
44+
- ".github/workflows/spark_sql_test_reusable.yml"
45+
- ".github/actions/setup-builder/**"
46+
- ".github/actions/setup-spark-builder/**"
47+
# On-demand PR runs: a committer adds the `run-spark-3.4-tests` label
48+
# and the workflow runs against the PR's merge ref. Works for forks.
49+
pull_request:
50+
types: [labeled]
51+
workflow_dispatch:
52+
inputs:
53+
collect-fallback-logs:
54+
description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
55+
required: false
56+
default: false
57+
type: boolean
58+
59+
jobs:
60+
spark-sql:
61+
if: github.event_name != 'pull_request' || github.event.label.name == 'run-spark-3.4-tests'
62+
uses: ./.github/workflows/spark_sql_test_reusable.yml
63+
with:
64+
spark-short: '3.4'
65+
spark-full: '3.4.3'
66+
java: 11
67+
collect-fallback-logs: ${{ github.event.inputs.collect-fallback-logs == 'true' }}

‎.github/workflows/spark_sql_test_3_5.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on every PR and on main. Spark 3.5 is the default supported version.
19+
name: Spark SQL Tests (Spark 3.5)
20+
21+
concurrency:
22+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23+
cancel-in-progress: true
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
paths:
30+
- "native/**/src/**"
31+
- "native/**/Cargo.toml"
32+
- "native/Cargo.lock"
33+
- "!native/hdfs/**"
34+
- "!native/fs-hdfs/**"
35+
- "common/src/main/**"
36+
- "common/pom.xml"
37+
- "spark/src/main/**"
38+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
39+
- "spark/pom.xml"
40+
- "dev/diffs/**"
41+
- "pom.xml"
42+
- "rust-toolchain.toml"
43+
- ".github/workflows/spark_sql_test_3_5.yml"
44+
- ".github/workflows/spark_sql_test_reusable.yml"
45+
- ".github/actions/setup-builder/**"
46+
- ".github/actions/setup-spark-builder/**"
47+
pull_request:
48+
paths:
49+
- "native/**/src/**"
50+
- "native/**/Cargo.toml"
51+
- "native/Cargo.lock"
52+
- "!native/hdfs/**"
53+
- "!native/fs-hdfs/**"
54+
- "common/src/main/**"
55+
- "common/pom.xml"
56+
- "spark/src/main/**"
57+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
58+
- "spark/pom.xml"
59+
- "dev/diffs/**"
60+
- "pom.xml"
61+
- "rust-toolchain.toml"
62+
- ".github/workflows/spark_sql_test_3_5.yml"
63+
- ".github/workflows/spark_sql_test_reusable.yml"
64+
- ".github/actions/setup-builder/**"
65+
- ".github/actions/setup-spark-builder/**"
66+
workflow_dispatch:
67+
inputs:
68+
collect-fallback-logs:
69+
description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
70+
required: false
71+
default: false
72+
type: boolean
73+
74+
jobs:
75+
spark-sql:
76+
uses: ./.github/workflows/spark_sql_test_reusable.yml
77+
with:
78+
spark-short: '3.5'
79+
spark-full: '3.5.8'
80+
java: 11
81+
collect-fallback-logs: ${{ github.event.inputs.collect-fallback-logs == 'true' }}

‎.github/workflows/spark_sql_test_4_0.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on every PR and on main. Spark 4.0 is the newest stable Spark line.
19+
name: Spark SQL Tests (Spark 4.0)
20+
21+
concurrency:
22+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23+
cancel-in-progress: true
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
paths:
30+
- "native/**/src/**"
31+
- "native/**/Cargo.toml"
32+
- "native/Cargo.lock"
33+
- "!native/hdfs/**"
34+
- "!native/fs-hdfs/**"
35+
- "common/src/main/**"
36+
- "common/pom.xml"
37+
- "spark/src/main/**"
38+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
39+
- "spark/pom.xml"
40+
- "dev/diffs/**"
41+
- "pom.xml"
42+
- "rust-toolchain.toml"
43+
- ".github/workflows/spark_sql_test_4_0.yml"
44+
- ".github/workflows/spark_sql_test_reusable.yml"
45+
- ".github/actions/setup-builder/**"
46+
- ".github/actions/setup-spark-builder/**"
47+
pull_request:
48+
paths:
49+
- "native/**/src/**"
50+
- "native/**/Cargo.toml"
51+
- "native/Cargo.lock"
52+
- "!native/hdfs/**"
53+
- "!native/fs-hdfs/**"
54+
- "common/src/main/**"
55+
- "common/pom.xml"
56+
- "spark/src/main/**"
57+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
58+
- "spark/pom.xml"
59+
- "dev/diffs/**"
60+
- "pom.xml"
61+
- "rust-toolchain.toml"
62+
- ".github/workflows/spark_sql_test_4_0.yml"
63+
- ".github/workflows/spark_sql_test_reusable.yml"
64+
- ".github/actions/setup-builder/**"
65+
- ".github/actions/setup-spark-builder/**"
66+
workflow_dispatch:
67+
inputs:
68+
collect-fallback-logs:
69+
description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
70+
required: false
71+
default: false
72+
type: boolean
73+
74+
jobs:
75+
spark-sql:
76+
uses: ./.github/workflows/spark_sql_test_reusable.yml
77+
with:
78+
spark-short: '4.0'
79+
spark-full: '4.0.2'
80+
java: 21
81+
collect-fallback-logs: ${{ github.event.inputs.collect-fallback-logs == 'true' }}

‎.github/workflows/spark_sql_test_4_1.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Runs on main only. Forward-looking coverage for the in-development Spark line.
19+
name: Spark SQL Tests (Spark 4.1)
20+
21+
concurrency:
22+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
23+
cancel-in-progress: true
24+
25+
on:
26+
push:
27+
branches:
28+
- main
29+
paths:
30+
- "native/**/src/**"
31+
- "native/**/Cargo.toml"
32+
- "native/Cargo.lock"
33+
- "!native/hdfs/**"
34+
- "!native/fs-hdfs/**"
35+
- "common/src/main/**"
36+
- "common/pom.xml"
37+
- "spark/src/main/**"
38+
- "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
39+
- "spark/pom.xml"
40+
- "dev/diffs/**"
41+
- "pom.xml"
42+
- "rust-toolchain.toml"
43+
- ".github/workflows/spark_sql_test_4_1.yml"
44+
- ".github/workflows/spark_sql_test_reusable.yml"
45+
- ".github/actions/setup-builder/**"
46+
- ".github/actions/setup-spark-builder/**"
47+
# On-demand PR runs: a committer adds the `run-spark-4.1-tests` label
48+
# and the workflow runs against the PR's merge ref. Works for forks.
49+
pull_request:
50+
types: [labeled]
51+
workflow_dispatch:
52+
inputs:
53+
collect-fallback-logs:
54+
description: 'Whether to collect Comet fallback reasons from spark sql unit test logs'
55+
required: false
56+
default: false
57+
type: boolean
58+
59+
jobs:
60+
spark-sql:
61+
if: github.event_name != 'pull_request' || github.event.label.name == 'run-spark-4.1-tests'
62+
uses: ./.github/workflows/spark_sql_test_reusable.yml
63+
with:
64+
spark-short: '4.1'
65+
spark-full: '4.1.1'
66+
java: 17
67+
collect-fallback-logs: ${{ github.event.inputs.collect-fallback-logs == 'true' }}

0 commit comments

Comments
 (0)