Skip to content

Commit 9103bb7

Browse files
committed
Remove old Spark 3.3 DBR shim sources
Signed-off-by: Gera Shegalov <gshegalov@nvidia.com>
1 parent 4ec5b87 commit 9103bb7

11 files changed

Lines changed: 96 additions & 243 deletions

File tree

sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/DatabricksShimServiceProvider.scala

Lines changed: 0 additions & 66 deletions
This file was deleted.
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Copyright (c) 2026, NVIDIA CORPORATION.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*** spark-rapids-shim-json-lines
18+
{"spark": "330db"}
19+
{"spark": "331"}
20+
{"spark": "332"}
21+
{"spark": "332db"}
22+
{"spark": "333"}
23+
{"spark": "334"}
24+
{"spark": "340"}
25+
{"spark": "341"}
26+
{"spark": "341db"}
27+
{"spark": "342"}
28+
{"spark": "343"}
29+
{"spark": "344"}
30+
{"spark": "350"}
31+
{"spark": "350db143"}
32+
{"spark": "351"}
33+
{"spark": "352"}
34+
{"spark": "353"}
35+
{"spark": "354"}
36+
{"spark": "355"}
37+
{"spark": "356"}
38+
{"spark": "357"}
39+
{"spark": "358"}
40+
{"spark": "400"}
41+
{"spark": "400db173"}
42+
{"spark": "401"}
43+
{"spark": "402"}
44+
{"spark": "411"}
45+
spark-rapids-shim-json-lines ***/
46+
package com.nvidia.spark.rapids.shims
47+
48+
import com.nvidia.spark.rapids.{
49+
ExprChecks,
50+
ExprRule,
51+
GpuCast,
52+
GpuExpression,
53+
GpuOverrides,
54+
TypeSig,
55+
UnaryExprMeta
56+
}
57+
58+
import org.apache.spark.sql.catalyst.expressions.{CheckOverflowInTableInsert, Expression}
59+
import org.apache.spark.sql.rapids.GpuCheckOverflowInTableInsert
60+
61+
object CheckOverflowInTableInsertShims {
62+
val exprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
63+
Seq(
64+
// Add expression CheckOverflowInTableInsert starting Spark-3.3.1+.
65+
// Accepts all types as input as the child Cast does the type checking and the calculations.
66+
GpuOverrides.expr[CheckOverflowInTableInsert](
67+
"Casting a numeric value as another numeric type in store assignment",
68+
ExprChecks.unaryProjectInputMatchesOutput(
69+
TypeSig.all,
70+
TypeSig.all),
71+
(t, conf, p, r) => new UnaryExprMeta[CheckOverflowInTableInsert](t, conf, p, r) {
72+
override def convertToGpu(child: Expression): GpuExpression = {
73+
child match {
74+
case c: GpuCast => GpuCheckOverflowInTableInsert(c, t.columnName)
75+
case _ =>
76+
throw new IllegalStateException("Expression child is not of Type GpuCast")
77+
}
78+
}
79+
})
80+
).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap
81+
}
82+
}

sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/Spark321PlusDBShims.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ trait Spark321PlusDBShims extends SparkShims
124124
TypeSig.all,
125125
Map("partitionSpec" ->
126126
new InputCheck(TypeSig.commonCudfTypes + TypeSig.NULL + TypeSig.DECIMAL_128,
127-
TypeSig.all, Nil))),
127+
TypeSig.all, List.empty))),
128128
(runningWindowFunctionExec, conf, p, r) =>
129129
new GpuRunningWindowExecMeta(runningWindowFunctionExec, conf, p, r)
130130
)

sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/Spark330PlusDBShims.scala

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,29 +30,16 @@ import org.apache.spark.sql.catalyst.plans.physical.SinglePartition
3030
import org.apache.spark.sql.execution.{ColumnarToRowTransition, SparkPlan}
3131
import org.apache.spark.sql.execution.adaptive.ShuffleQueryStageExec
3232
import org.apache.spark.sql.execution.exchange.{EXECUTOR_BROADCAST, ShuffleExchangeExec, ShuffleExchangeLike}
33-
import org.apache.spark.sql.rapids.{GpuCheckOverflowInTableInsert, GpuElementAtMeta}
33+
import org.apache.spark.sql.rapids.GpuElementAtMeta
3434
import org.apache.spark.sql.rapids.execution.{GpuBroadcastHashJoinExec, GpuBroadcastNestedLoopJoinExec}
3535

3636
trait Spark330PlusDBShims extends Spark321PlusDBShims {
3737
override def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
3838
val shimExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = Seq(
39-
GpuOverrides.expr[CheckOverflowInTableInsert](
40-
"Casting a numeric value as another numeric type in store assignment",
41-
ExprChecks.unaryProjectInputMatchesOutput(
42-
TypeSig.all,
43-
TypeSig.all),
44-
(t, conf, p, r) => new UnaryExprMeta[CheckOverflowInTableInsert](t, conf, p, r) {
45-
override def convertToGpu(child: Expression): GpuExpression = {
46-
child match {
47-
case c: GpuCast => GpuCheckOverflowInTableInsert(c, t.columnName)
48-
case _ =>
49-
throw new IllegalStateException("Expression child is not of Type GpuCast")
50-
}
51-
}
52-
}),
5339
GpuElementAtMeta.elementAtRule(true)
5440
).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap
55-
super.getExprs ++ shimExprs ++ DayTimeIntervalShims.exprs ++ RoundingShims.exprs
41+
super.getExprs ++ CheckOverflowInTableInsertShims.exprs ++ shimExprs ++
42+
DayTimeIntervalShims.exprs ++ RoundingShims.exprs
5643
}
5744

5845
override def getExecs: Map[Class[_ <: SparkPlan], ExecRule[_ <: SparkPlan]] =

sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/SparkShims.scala

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2022-2023, NVIDIA CORPORATION.
2+
* Copyright (c) 2022-2026, NVIDIA CORPORATION.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -22,17 +22,18 @@ package com.nvidia.spark.rapids.shims
2222
import com.nvidia.spark.rapids._
2323

2424
import org.apache.spark.sql.catalyst.expressions.Expression
25-
import org.apache.spark.sql.execution.command.{CreateDataSourceTableAsSelectCommand, DataWritingCommand, RunnableCommand}
25+
import org.apache.spark.sql.execution.command.{DataWritingCommand, RunnableCommand}
2626

2727
object SparkShimImpl extends Spark330PlusDBShims {
2828
// AnsiCast is removed from Spark3.4.0
2929
override def ansiCastRule: ExprRule[_ <: Expression] = null
3030

3131
override def getDataWriteCmds: Map[Class[_ <: DataWritingCommand],
3232
DataWritingCommandRule[_ <: DataWritingCommand]] = {
33-
Seq(GpuOverrides.dataWriteCmd[CreateDataSourceTableAsSelectCommand](
34-
"Create table with select command",
35-
(a, conf, p, r) => new CreateDataSourceTableAsSelectCommandMeta(a, conf, p, r))
33+
Seq(
34+
GpuOverrides.dataWriteCmdFromShim(
35+
CreateDataSourceTableAsSelectRules.dataWriteCmd,
36+
(a, conf, p, r) => new CreateDataSourceTableAsSelectCommandMeta(a, conf, p, r))
3637
).map(r => (r.getClassFor.asSubclass(classOf[DataWritingCommand]), r)).toMap
3738
}
3839

sql-plugin/src/main/spark330db/scala/com/nvidia/spark/rapids/shims/spark330db/SparkShimServiceProvider.scala

Lines changed: 0 additions & 37 deletions
This file was deleted.

sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/arithmetic.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ import com.nvidia.spark.rapids.Arm.{closeOnExcept, withResource}
4949
import com.nvidia.spark.rapids.RapidsPluginImplicits._
5050
import com.nvidia.spark.rapids.shims.NullIntolerantShim
5151

52-
import org.apache.spark.internal.Logging
5352
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
5453
import org.apache.spark.sql.catalyst.expressions.Expression
5554
import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}
@@ -279,7 +278,7 @@ case class GpuDecimalRemainder(
279278
left: Expression,
280279
right: Expression,
281280
failOnError: Boolean = SQLConf.get.ansiEnabled)
282-
extends GpuRemainderBase(left, right) with Logging {
281+
extends GpuRemainderBase(left, right) {
283282

284283
// scalastyle:off
285284
// The formula follows Hive which is based on the SQL standard and MS SQL:

sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupedPythonRunnerFactory.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2023-2025, NVIDIA CORPORATION.
2+
* Copyright (c) 2023-2026, NVIDIA CORPORATION.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -25,14 +25,14 @@ import org.apache.spark.sql.rapids.shims.ArrowUtilsShim
2525
import org.apache.spark.sql.types.StructType
2626
import org.apache.spark.sql.vectorized.ColumnarBatch
2727

28-
case class GpuGroupedPythonRunnerFactory(
28+
class GpuGroupedPythonRunnerFactory(
2929
conf: org.apache.spark.sql.internal.SQLConf,
3030
chainedFunc: Seq[(ChainedPythonFunctions, Long)],
3131
argOffsets: Array[Array[Int]],
3232
dedupAttrs: StructType,
3333
pythonOutputSchema: StructType,
3434
evalType: Int,
35-
argNames: Option[Array[Array[Option[String]]]] = None) {
35+
argNames: Option[Array[Array[Option[String]]]]) extends Serializable {
3636
// Configs from DB runtime
3737
val maxBytes = conf.pandasZeroConfConversionGroupbyApplyMaxBytesPerSlice
3838
val zeroConfEnabled = conf.pandasZeroConfConversionGroupbyApplyEnabled

sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/OriginContextShim.scala

Lines changed: 0 additions & 33 deletions
This file was deleted.

sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/shims/SparkDateTimeExceptionShims.scala

Lines changed: 0 additions & 38 deletions
This file was deleted.

0 commit comments

Comments
 (0)