Skip to content

Commit 348342d

Browse files
committed
add hdfs test
1 parent 162914f commit 348342d

4 files changed

Lines changed: 14 additions & 2 deletions

File tree

.github/workflows/pr_build_linux.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,9 @@ jobs:
183183
suites: |
184184
org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite
185185
org.apache.comet.parquet.ParquetReadFromHdfsSuite
186+
- value: "hdfs"
187+
suites: |
188+
org.apache.comet.parquet.ParquetReadFromHdfsSuite
186189
fail-fast: false
187190
name: ${{ matrix.os }}/java ${{ matrix.java_version }}-features [${{ matrix.features.value }}]
188191
runs-on: ${{ matrix.os }}
@@ -203,6 +206,6 @@ jobs:
203206
with:
204207
artifact_name: ${{ matrix.os }}-java-${{ matrix.java_version }}-features-${{ matrix.features.value }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
205208
features: ${{ matrix.features.value }}
206-
maven_opts: "-Dtest=none"
209+
maven_opts: "-Dtest=none -Dfeatures=${{ matrix.features.value }}"
207210
suites: ${{ matrix.features.suites }}
208211
upload-test-reports: true

spark/src/test/scala/org/apache/comet/parquet/ParquetReadFromFakeHadoopFsSuite.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,9 @@ class ParquetReadFromFakeHadoopFsSuite extends CometTestBase with AdaptiveSparkP
7474
.startsWith(FakeHDFSFileSystem.PREFIX))
7575
}
7676

77-
ignore("test native_datafusion scan on fake fs") {
77+
test("test native_datafusion scan on fake fs") {
78+
assume(featureEnabled("hdfs-opendal"))
79+
7880
val testFilePath =
7981
s"${FakeHDFSFileSystem.PREFIX}${fake_root_dir.getAbsolutePath}/data/test-file.parquet"
8082
writeTestParquetFile(testFilePath)

spark/src/test/scala/org/apache/comet/parquet/ParquetReadFromHdfsSuite.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ class ParquetReadFromHdfsSuite
6363
}
6464

6565
test("test native_datafusion scan on hdfs") {
66+
assume(featureEnabled("hdfs") || featureEnabled("hdfs-opendal"))
67+
6668
withTmpHdfsDir { dir =>
6769
{
6870
val testFilePath = dir.toString

spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1138,4 +1138,9 @@ abstract class CometTestBase
11381138
usingDataSourceExec(conf) &&
11391139
!CometConf.COMET_SCAN_ALLOW_INCOMPATIBLE.get(conf)
11401140
}
1141+
1142+
def featureEnabled(feature: String): Boolean = {
1143+
System.getProperty("feature", "").split(",").contains(feature)
1144+
}
1145+
11411146
}

0 commit comments

Comments
 (0)