Skip to content

Commit fa067f7

Browse files
nightscapenightscape
committed
fix: Fix 3.5.0 compile issues
1 parent cda2140 commit fa067f7

File tree

7 files changed

+211
-6
lines changed

7 files changed

+211
-6
lines changed

‎build.sc

+18-2
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,24 @@ trait SparkModule extends Cross.Module2[String, String] with SbtModule with CiRe
1212
override def millSourcePath = super.millSourcePath / os.up
1313

1414
// Custom source layout for Spark Data Source API 2
15-
val sparkVersionSpecificSources = if (sparkVersion >= "3.4.0") {
16-
Seq("scala", "3.0_and_up/scala", "3.1_and_up/scala", "3.2_and_up/scala", "3.3_and_up/scala", "3.4_and_up/scala")
15+
val sparkVersionSpecificSources = if (sparkVersion >= "3.5.0") {
16+
Seq(
17+
"scala",
18+
"3.0_and_up/scala",
19+
"3.1_and_up/scala",
20+
"3.2_and_up/scala",
21+
"3.3_and_up/scala",
22+
"3.5_and_up/scala"
23+
)
24+
} else if (sparkVersion >= "3.4.0") {
25+
Seq(
26+
"scala",
27+
"3.0_and_up/scala",
28+
"3.1_and_up/scala",
29+
"3.2_and_up/scala",
30+
"3.3_and_up/scala",
31+
"3.4_and_up/scala"
32+
)
1733
} else if (sparkVersion >= "3.3.0") {
1834
Seq(
1935
"scala",

‎src/main/2.4/scala/com/crealytics/spark/excel/v2/ExcelParserBase.scala

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright 2022 Martin Mauch (@nightscape)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.crealytics.spark.excel.v2
18+
19+
import _root_.org.apache.spark.sql.catalyst.util.BadRecordException
20+
import org.apache.spark.unsafe.types.UTF8String
21+
import org.apache.spark.sql.catalyst.InternalRow
22+
23+
trait ExcelParserBase {
24+
25+
protected def getCurrentInput: UTF8String
26+
def badRecord(partialResults: Array[InternalRow], baseException: Throwable): BadRecordException =
27+
BadRecordException(() => getCurrentInput, () => partialResults.headOption, baseException)
28+
}

‎src/main/3.0_3.1_3.2_3.3/scala/com/crealytics/spark/excel/v2/ExcelOptions.scala renamed to ‎src/main/3.0_and_up/scala/com/crealytics/spark/excel/v2/ExcelOptions.scala

File renamed without changes.

‎src/main/3.0_and_up/scala/com/crealytics/spark/excel/v2/ExcelParserBase.scala

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright 2022 Martin Mauch (@nightscape)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.crealytics.spark.excel.v2
18+
19+
import _root_.org.apache.spark.sql.catalyst.util.BadRecordException
20+
import org.apache.spark.unsafe.types.UTF8String
21+
import org.apache.spark.sql.catalyst.InternalRow
22+
23+
trait ExcelParserBase {
24+
25+
protected def getCurrentInput: UTF8String
26+
def badRecord(partialResults: Array[InternalRow], baseException: Throwable): BadRecordException =
27+
BadRecordException(() => getCurrentInput, () => partialResults.headOption, baseException)
28+
}

‎src/main/3.5_and_up/scala/com/crealytics/spark/excel/v2/ExcelParserBase.scala

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright 2022 Martin Mauch (@nightscape)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.crealytics.spark.excel.v2
18+
19+
import _root_.org.apache.spark.sql.catalyst.util.BadRecordException
20+
import org.apache.spark.unsafe.types.UTF8String
21+
import org.apache.spark.sql.catalyst.InternalRow
22+
23+
trait ExcelParserBase {
24+
25+
protected def getCurrentInput: UTF8String
26+
def badRecord(partialResults: Array[InternalRow], baseException: Throwable): BadRecordException =
27+
BadRecordException(() => getCurrentInput, () => partialResults, baseException)
28+
}

‎src/main/3.5_and_up/scala/com/crealytics/spark/excel/v2/ExcelPartitionReaderFactory.scala

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* Copyright 2022 Martin Mauch (@nightscape)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.crealytics.spark.excel.v2
18+
19+
import org.apache.hadoop.conf.Configuration
20+
import org.apache.spark.broadcast.Broadcast
21+
import org.apache.spark.sql.catalyst.{InternalRow, FileSourceOptions}
22+
import org.apache.spark.sql.connector.read.PartitionReader
23+
import org.apache.spark.sql.execution.datasources.PartitionedFile
24+
import org.apache.spark.sql.execution.datasources.v2._
25+
import org.apache.spark.sql.internal.SQLConf
26+
import org.apache.spark.sql.sources.Filter
27+
import org.apache.spark.sql.types.StructType
28+
import org.apache.spark.util.SerializableConfiguration
29+
30+
import java.net.URI
31+
import scala.util.control.NonFatal
32+
33+
/** A factory used to create Excel readers.
34+
*
35+
* @param sqlConf
36+
* SQL configuration.
37+
* @param broadcastedConf
38+
* Broadcasted serializable Hadoop Configuration.
39+
* @param dataSchema
40+
* Schema of Excel files.
41+
* @param readDataSchema
42+
* Required data schema in the batch scan.
43+
* @param partitionSchema
44+
* Schema of partitions.
45+
* @param parsedOptions
46+
* Options for parsing Excel files.
47+
*/
48+
case class ExcelPartitionReaderFactory(
49+
sqlConf: SQLConf,
50+
broadcastedConf: Broadcast[SerializableConfiguration],
51+
dataSchema: StructType,
52+
readDataSchema: StructType,
53+
partitionSchema: StructType,
54+
parsedOptions: ExcelOptions,
55+
filters: Seq[Filter]
56+
) extends FilePartitionReaderFactory {
57+
protected def options: FileSourceOptions = new FileSourceOptions(Map(
58+
FileSourceOptions.IGNORE_CORRUPT_FILES -> "true",
59+
FileSourceOptions.IGNORE_MISSING_FILES -> "true"
60+
))
61+
override def buildReader(file: PartitionedFile): PartitionReader[InternalRow] = {
62+
val conf = broadcastedConf.value.value
63+
val actualDataSchema =
64+
StructType(dataSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord))
65+
val actualReadDataSchema =
66+
StructType(readDataSchema.filterNot(_.name == parsedOptions.columnNameOfCorruptRecord))
67+
val parser = new ExcelParser(actualDataSchema, actualReadDataSchema, parsedOptions, filters)
68+
val headerChecker =
69+
new ExcelHeaderChecker(actualReadDataSchema, parsedOptions, source = s"Excel file: ${file.filePath}")
70+
val iter = readFile(conf, file, parser, headerChecker, readDataSchema)
71+
val partitionReader = new SparkExcelPartitionReaderFromIterator(iter)
72+
new PartitionReaderWithPartitionValues(partitionReader, readDataSchema, partitionSchema, file.partitionValues)
73+
}
74+
75+
private def readFile(
76+
conf: Configuration,
77+
file: PartitionedFile,
78+
parser: ExcelParser,
79+
headerChecker: ExcelHeaderChecker,
80+
requiredSchema: StructType
81+
): SheetData[InternalRow] = {
82+
val excelHelper = ExcelHelper(parsedOptions)
83+
val sheetData = excelHelper.getSheetData(conf, URI.create(file.filePath.toString))
84+
try {
85+
SheetData(
86+
ExcelParser.parseIterator(sheetData.rowIterator, parser, headerChecker, requiredSchema),
87+
sheetData.resourcesToClose
88+
)
89+
} catch {
90+
case NonFatal(t) => {
91+
sheetData.close()
92+
throw t
93+
}
94+
}
95+
}
96+
97+
}
98+
99+
private class SparkExcelPartitionReaderFromIterator(sheetData: SheetData[InternalRow])
100+
extends PartitionReaderFromIterator[InternalRow](sheetData.rowIterator) {
101+
override def close(): Unit = {
102+
super.close()
103+
sheetData.close()
104+
}
105+
}

‎src/main/scala/com/crealytics/spark/excel/v2/ExcelParser.scala

+4-4
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ import org.apache.poi.ss.usermodel.DateUtil
4242
* The pushdown filters that should be applied to converted values.
4343
*/
4444
class ExcelParser(dataSchema: StructType, requiredSchema: StructType, val options: ExcelOptions, filters: Seq[Filter])
45-
extends Logging {
45+
extends Logging with ExcelParserBase {
4646
require(
4747
requiredSchema.toSet.subsetOf(dataSchema.toSet),
4848
s"requiredSchema (${requiredSchema.catalogString}) should be the subset of " +
@@ -95,7 +95,7 @@ class ExcelParser(dataSchema: StructType, requiredSchema: StructType, val option
9595
private val pushedFilters = new ExcelFilters(filters, requiredSchema)
9696

9797
/* Retrieve the raw record string. */
98-
private def getCurrentInput: UTF8String = UTF8String
98+
protected def getCurrentInput: UTF8String = UTF8String
9999
.fromString("TODO: how to show the corrupted record?")
100100

101101
/** This parser first picks some tokens from the input tokens, according to the required schema, then parse these
@@ -352,7 +352,7 @@ class ExcelParser(dataSchema: StructType, requiredSchema: StructType, val option
352352

353353
private def convert(tokens: Vector[Cell]): Option[InternalRow] = {
354354
if (tokens == null) {
355-
throw BadRecordException(() => getCurrentInput, () => None, new RuntimeException("Malformed Excel record"))
355+
throw badRecord(Array.empty, new RuntimeException("Malformed Excel record"))
356356
}
357357

358358
var badRecordException: Option[Throwable] =
@@ -396,7 +396,7 @@ class ExcelParser(dataSchema: StructType, requiredSchema: StructType, val option
396396
if (skipRow) { noRows }
397397
else {
398398
if (badRecordException.isDefined) {
399-
throw BadRecordException(() => getCurrentInput, () => requiredRow.headOption, badRecordException.get)
399+
throw badRecord(requiredRow.toArray, badRecordException.get)
400400
} else { requiredRow }
401401
}
402402
}

0 commit comments

Comments
 (0)