Skip to content

Commit 6d4638e

Browse files
viiryaIsaac
andcommitted
[SPARK-60118][SQL] Fix Int overflow in the fast-fail check of casting string to decimal
Compute the number of integral digits in `Decimal.fromString`/`fromStringANSI` as a Long, so that strings with an exponent near Int.MaxValue (e.g. '1e2147483647') are rejected by the fast-fail check instead of failing with a raw java.lang.ArithmeticException. Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 215a146 commit 6d4638e

4 files changed

Lines changed: 24 additions & 2 deletions

File tree

‎sql/api/src/main/scala/org/apache/spark/sql/types/Decimal.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -635,8 +635,9 @@ object Decimal {
635635
}
636636
}
637637

638-
private def numDigitsInIntegralPart(bigDecimal: JavaBigDecimal): Int =
639-
bigDecimal.precision - bigDecimal.scale
638+
// Computed in Long since `precision - scale` can overflow Int, e.g. for "1e2147483647".
639+
private def numDigitsInIntegralPart(bigDecimal: JavaBigDecimal): Long =
640+
bigDecimal.precision.toLong - bigDecimal.scale
640641

641642
private def stringToJavaBigDecimal(str: UTF8String): JavaBigDecimal = {
642643
// According the benchmark test, `s.toString.trim` is much faster than `s.trim.toString`.

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOffSuite.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -559,6 +559,9 @@ class CastWithAnsiOffSuite extends CastSuiteBase {
559559
Decimal("60000000000000000000000000000000000000"))
560560
checkEvaluation(cast("6E+38", DecimalType(38, 0)), null)
561561
checkEvaluation(cast("6E+37", DecimalType(38, 1)), null)
562+
Seq("1e2147483647", "0e2147483647", "1.5e2147483647", "1e2147483648").foreach { s =>
563+
checkEvaluation(cast(s, DecimalType(10, 2)), null)
564+
}
562565

563566
checkEvaluation(cast("abcd", DecimalType(38, 1)), null)
564567
}

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,11 @@ class CastWithAnsiOnSuite extends CastSuiteBase with QueryErrorsBase {
475475
checkExceptionInExpression[ArithmeticException](
476476
cast("6E+37", DecimalType(38, 1)),
477477
"cannot be represented as Decimal(38, 1)")
478+
Seq("1e2147483647", "0e2147483647", "1.5e2147483647", "1e2147483648").foreach { s =>
479+
checkExceptionInExpression[ArithmeticException](
480+
cast(s, DecimalType(10, 2)),
481+
"NUMERIC_OUT_OF_SUPPORTED_RANGE")
482+
}
478483

479484
checkExceptionInExpression[NumberFormatException](
480485
cast("abcd", DecimalType(38, 1)),

‎sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,19 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester with SQLHelper
446446
}
447447
}
448448

449+
test("UTF8String to Decimal with exponent near Int.MaxValue") {
450+
// The number of integral digits (precision - scale) of these values overflows Int.
451+
Seq("1e2147483647", "-1E+2147483647", "0e2147483647", "12e2147483647",
452+
"1.5e2147483647", "1e2147483648").foreach { string =>
453+
assert(Decimal.fromString(UTF8String.fromString(string)) === null)
454+
checkError(
455+
exception = intercept[SparkArithmeticException](
456+
Decimal.fromStringANSI(UTF8String.fromString(string))),
457+
condition = "NUMERIC_OUT_OF_SUPPORTED_RANGE",
458+
parameters = Map("value" -> string))
459+
}
460+
}
461+
449462
// 18 is a max number of digits in Decimal's compact long
450463
test("SPARK-41554: decrease/increase scale by 18 and more on compact decimal") {
451464
val unscaledNums = Seq(

0 commit comments

Comments
 (0)