Skip to content

Commit dd6902a

Browse files
committed
Remove old Spark 3.5 shim sources
Signed-off-by: Gera Shegalov <gshegalov@nvidia.com>
1 parent e4f3ab7 commit dd6902a

15 files changed

Lines changed: 18 additions & 424 deletions

File tree

sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/LegacyBehaviorPolicyShim.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@
3333
spark-rapids-shim-json-lines ***/
3434
package com.nvidia.spark.rapids.shims
3535

36-
import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf}
36+
import org.apache.spark.sql.internal.SQLConf
3737

3838
object LegacyBehaviorPolicyShim {
39-
val CORRECTED_STR: String = LegacyBehaviorPolicy.CORRECTED.toString
40-
val EXCEPTION_STR: String = LegacyBehaviorPolicy.EXCEPTION.toString
39+
val CORRECTED_STR: String = "CORRECTED"
40+
val EXCEPTION_STR: String = "EXCEPTION"
4141

4242
def isLegacyTimeParserPolicy(): Boolean = {
43-
SQLConf.get.legacyTimeParserPolicy == LegacyBehaviorPolicy.LEGACY
43+
SQLConf.get.legacyTimeParserPolicy.toString == "LEGACY"
4444
}
4545
}

sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/NullOutputStreamShim.scala

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,11 @@
3333
spark-rapids-shim-json-lines ***/
3434
package com.nvidia.spark.rapids.shims
3535

36-
import org.apache.commons.io.output.NullOutputStream
36+
import java.io.OutputStream
3737

3838
object NullOutputStreamShim {
39-
def INSTANCE = NullOutputStream.INSTANCE
39+
val INSTANCE: OutputStream = new OutputStream {
40+
override def write(b: Int): Unit = {}
41+
override def write(b: Array[Byte], off: Int, len: Int): Unit = {}
42+
}
4043
}

sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/Spark350PlusNonDBShims.scala

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ trait Spark350PlusNonDBShims extends Spark340PlusNonDBShims {
130130
// plugin is also an union of all the types of Pandas UDF.
131131
(TypeSig.commonCudfTypes + TypeSig.ARRAY).nested() + TypeSig.STRUCT,
132132
TypeSig.unionOfPandasUdfOut,
133-
repeatingParamCheck = Some(RepeatingParamCheck(
133+
repeatingParamCheck = Some(new RepeatingParamCheck(
134134
"param",
135135
(TypeSig.commonCudfTypes + TypeSig.ARRAY + TypeSig.STRUCT).nested(),
136136
TypeSig.all))),
@@ -150,24 +150,24 @@ trait Spark350PlusNonDBShims extends Spark340PlusNonDBShims {
150150
ExprChecks.projectOnly(
151151
TypeSig.all,
152152
TypeSig.all,
153-
Seq(ParamCheck("condition", TypeSig.all, TypeSig.all)),
154-
Some(RepeatingParamCheck("outputs", TypeSig.all, TypeSig.all))
153+
Seq(new ParamCheck("condition", TypeSig.all, TypeSig.all)),
154+
Some(new RepeatingParamCheck("outputs", TypeSig.all, TypeSig.all))
155155
),
156156
(keep, conf, p, r) => new GpuKeepInstructionMeta(keep, conf, p, r)),
157157
GpuOverrides.expr[Discard](
158158
"Discard instruction for MERGE operations - discards rows based on condition",
159159
ExprChecks.projectOnly(
160160
TypeSig.all,
161161
TypeSig.all,
162-
Seq(ParamCheck("condition", TypeSig.all, TypeSig.all))),
162+
Seq(new ParamCheck("condition", TypeSig.all, TypeSig.all))),
163163
(discard, conf, p, r) => new GpuDiscardInstructionMeta(discard, conf, p, r)),
164164
GpuOverrides.expr[Split](
165165
"Split instruction for MERGE operations - splits rows into multiple outputs",
166166
ExprChecks.projectOnly(
167167
TypeSig.all,
168168
TypeSig.all,
169-
Seq(ParamCheck("condition", TypeSig.all, TypeSig.all)),
170-
Some(RepeatingParamCheck("outputs", TypeSig.all, TypeSig.all))),
169+
Seq(new ParamCheck("condition", TypeSig.all, TypeSig.all)),
170+
Some(new RepeatingParamCheck("outputs", TypeSig.all, TypeSig.all))),
171171
(split, conf, p, r) => new GpuSplitInstructionMeta(split, conf, p, r))
172172
).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap
173173
super.getExprs ++ shimExprs

sql-plugin/src/main/spark350/scala/com/nvidia/spark/rapids/shims/spark350/SparkShimServiceProvider.scala

Lines changed: 0 additions & 36 deletions
This file was deleted.

sql-plugin/src/main/spark350db143/scala/com/nvidia/spark/rapids/shims/spark350db143/SparkShimServiceProvider.scala

Lines changed: 0 additions & 51 deletions
This file was deleted.

sql-plugin/src/main/spark350db143/scala/org/apache/spark/sql/rapids/execution/python/shims/PythonArgumentsUtils.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,15 @@ object PythonArgumentUtils {
4848
(None, e)
4949
}
5050
if (allInputs.exists(_.semanticEquals(value))) {
51-
GpuArgumentMeta(allInputs.indexWhere(_.semanticEquals(value)), key)
51+
new GpuArgumentMeta(allInputs.indexWhere(_.semanticEquals(value)), key)
5252
} else {
5353
allInputs += value
5454
dataTypes += value.dataType
55-
GpuArgumentMeta(allInputs.length - 1, key)
55+
new GpuArgumentMeta(allInputs.length - 1, key)
5656
}
5757
}.toArray
5858
}.toArray
59-
GpuPythonArguments(allInputs.toSeq, dataTypes.toSeq,
59+
new GpuPythonArguments(allInputs.toSeq, dataTypes.toSeq,
6060
argMetas.map(_.map(_.offset)), Some(argMetas.map(_.map(_.name))))
6161
}
6262
}

sql-plugin/src/main/spark350db143/scala/org/apache/spark/sql/rapids/shims/SequenceSizeExceededLimitErrorBuilder.scala

Lines changed: 0 additions & 34 deletions
This file was deleted.

sql-plugin/src/main/spark351/scala/com/nvidia/spark/rapids/shims/spark351/SparkShimServiceProvider.scala

Lines changed: 0 additions & 36 deletions
This file was deleted.

sql-plugin/src/main/spark352/scala/com/nvidia/spark/rapids/shims/spark352/SparkShimServiceProvider.scala

Lines changed: 0 additions & 36 deletions
This file was deleted.

sql-plugin/src/main/spark353/scala/com/nvidia/spark/rapids/shims/spark353/SparkShimServiceProvider.scala

Lines changed: 0 additions & 36 deletions
This file was deleted.

0 commit comments

Comments
 (0)