[SPARK-31237][SQL][TESTS] Replace 3-letter time zones by zone offsets

In the PR, I propose to add a few `ZoneId` constant values to the `DateTimeTestUtils` object, and reuse the constants in tests. Proposed the following constants:
- PST = -08:00
- UTC = +00:00
- CEST = +02:00
- CET = +01:00
- JST = +09:00
- MIT = -09:30
- LA = America/Los_Angeles

All proposed constant values (except `LA`) are initialized by zone offsets according to their definitions. This will allow to avoid:
- Using of 3-letter time zones that have been already deprecated in JDK, see _Three-letter time zone IDs_ in https://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html
- Incorrect mapping of 3-letter time zones to zone offsets, see SPARK-31237. For example, `PST` is mapped to `America/Los_Angeles` instead of the `-08:00` zone offset.

Also this should improve stability and maintainability of test suites.

No

By running affected test suites.

Closes apache#28001 from MaxGekk/replace-pst.

Authored-by: Maxim Gekk <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit cec9604)
Signed-off-by: Wenchen Fan <[email protected]>

Author: MaxGekk

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala

@@ -28,7 +28,7 @@ import org.apache.spark.sql.types._
 class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
 
   test("String fields types are inferred correctly from null types") {
-    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "GMT")
+    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(NullType, "") == NullType)
@@ -48,7 +48,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("String fields types are inferred correctly from other types") {
-    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "GMT")
+    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(LongType, "1.0") == DoubleType)
@@ -69,18 +69,18 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Timestamp field types are inferred correctly via custom data format") {
-    var options = new CSVOptions(Map("timestampFormat" -> "yyyy-mm"), false, "GMT")
+    var options = new CSVOptions(Map("timestampFormat" -> "yyyy-mm"), false, "UTC")
     var inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(TimestampType, "2015-08") == TimestampType)
 
-    options = new CSVOptions(Map("timestampFormat" -> "yyyy"), false, "GMT")
+    options = new CSVOptions(Map("timestampFormat" -> "yyyy"), false, "UTC")
     inferSchema = new CSVInferSchema(options)
     assert(inferSchema.inferField(TimestampType, "2015") == TimestampType)
   }
 
   test("Timestamp field types are inferred correctly from other types") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(IntegerType, "2015-08-20 14") == StringType)
@@ -89,15 +89,15 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Boolean fields types are inferred correctly from other types") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(LongType, "Fale") == StringType)
     assert(inferSchema.inferField(DoubleType, "TRUEe") == StringType)
   }
 
   test("Type arrays are merged to highest common type") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(
@@ -112,14 +112,14 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Null fields are handled properly when a nullValue is specified") {
-    var options = new CSVOptions(Map("nullValue" -> "null"), false, "GMT")
+    var options = new CSVOptions(Map("nullValue" -> "null"), false, "UTC")
     var inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(NullType, "null") == NullType)
     assert(inferSchema.inferField(StringType, "null") == StringType)
     assert(inferSchema.inferField(LongType, "null") == LongType)
 
-    options = new CSVOptions(Map("nullValue" -> "\\N"), false, "GMT")
+    options = new CSVOptions(Map("nullValue" -> "\\N"), false, "UTC")
     inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(IntegerType, "\\N") == IntegerType)
@@ -130,22 +130,22 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Merging Nulltypes should yield Nulltype.") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     val mergedNullTypes = inferSchema.mergeRowTypes(Array(NullType), Array(NullType))
     assert(mergedNullTypes.sameElements(Array(NullType)))
   }
 
   test("SPARK-18433: Improve DataSource option keys to be more case-insensitive") {
-    val options = new CSVOptions(Map("TiMeStampFormat" -> "yyyy-mm"), false, "GMT")
+    val options = new CSVOptions(Map("TiMeStampFormat" -> "yyyy-mm"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(TimestampType, "2015-08") == TimestampType)
   }
 
   test("SPARK-18877: `inferField` on DecimalType should find a common type with `typeSoFar`") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     withSQLConf(SQLConf.LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED.key -> "true") {
@@ -166,7 +166,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
 
   test("DoubleType should be inferred when user defined nan/inf are provided") {
     val options = new CSVOptions(Map("nanValue" -> "nan", "negativeInf" -> "-inf",
-      "positiveInf" -> "inf"), false, "GMT")
+      "positiveInf" -> "inf"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(NullType, "nan") == DoubleType)
@@ -179,7 +179,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
       val options = new CSVOptions(
         parameters = Map("locale" -> langTag, "inferSchema" -> "true", "sep" -> "|"),
         columnPruning = false,
-        defaultTimeZoneId = "GMT")
+        defaultTimeZoneId = "UTC")
       val inferSchema = new CSVInferSchema(options)
 
       val df = new DecimalFormat("", new DecimalFormatSymbols(Locale.forLanguageTag(langTag)))

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala

@@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.csv
 
 import java.math.BigDecimal
 import java.text.{DecimalFormat, DecimalFormatSymbols}
-import java.time.ZoneOffset
 import java.util.{Locale, TimeZone}
 
 import org.apache.commons.lang3.time.FastDateFormat
@@ -44,7 +43,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
 
     stringValues.zip(decimalValues).foreach { case (strVal, decimalVal) =>
       val decimalValue = new BigDecimal(decimalVal.toString)
-      val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+      val options = new CSVOptions(Map.empty[String, String], false, "UTC")
       val parser = new UnivocityParser(StructType(Seq.empty), options)
       assert(parser.makeConverter("_1", decimalType).apply(strVal) ===
         Decimal(decimalValue, decimalType.precision, decimalType.scale))
@@ -58,22 +57,22 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     // Nullable field with nullValue option.
     types.foreach { t =>
       // Tests that a custom nullValue.
-      val nullValueOptions = new CSVOptions(Map("nullValue" -> "-"), false, "GMT")
+      val nullValueOptions = new CSVOptions(Map("nullValue" -> "-"), false, "UTC")
       var parser = new UnivocityParser(StructType(Seq.empty), nullValueOptions)
       val converter = parser.makeConverter("_1", t, nullable = true)
       assertNull(converter.apply("-"))
       assertNull(converter.apply(null))
 
       // Tests that the default nullValue is empty string.
-      val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+      val options = new CSVOptions(Map.empty[String, String], false, "UTC")
       parser = new UnivocityParser(StructType(Seq.empty), options)
       assertNull(parser.makeConverter("_1", t, nullable = true).apply(""))
     }
 
     // Not nullable field with nullValue option.
     types.foreach { t =>
       // Casts a null to not nullable field should throw an exception.
-      val options = new CSVOptions(Map("nullValue" -> "-"), false, "GMT")
+      val options = new CSVOptions(Map("nullValue" -> "-"), false, "UTC")
       val parser = new UnivocityParser(StructType(Seq.empty), options)
       val converter = parser.makeConverter("_1", t, nullable = false)
       var message = intercept[RuntimeException] {
@@ -89,15 +88,15 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     // If nullValue is different with empty string, then, empty string should not be casted into
     // null.
     Seq(true, false).foreach { b =>
-      val options = new CSVOptions(Map("nullValue" -> "null"), false, "GMT")
+      val options = new CSVOptions(Map("nullValue" -> "null"), false, "UTC")
       val parser = new UnivocityParser(StructType(Seq.empty), options)
       val converter = parser.makeConverter("_1", StringType, nullable = b)
       assert(converter.apply("") == UTF8String.fromString(""))
     }
   }
 
   test("Throws exception for empty string with non null type") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val exception = intercept[RuntimeException]{
       parser.makeConverter("_1", IntegerType, nullable = false).apply("")
@@ -106,7 +105,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Types are cast correctly") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     var parser = new UnivocityParser(StructType(Seq.empty), options)
     assert(parser.makeConverter("_1", ByteType).apply("10") == 10)
     assert(parser.makeConverter("_1", ShortType).apply("10") == 10)
@@ -117,7 +116,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     assert(parser.makeConverter("_1", BooleanType).apply("true") == true)
 
     var timestampsOptions =
-      new CSVOptions(Map("timestampFormat" -> "dd/MM/yyyy HH:mm"), false, "GMT")
+      new CSVOptions(Map("timestampFormat" -> "dd/MM/yyyy HH:mm"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), timestampsOptions)
     val customTimestamp = "31/01/2015 00:00"
     var format = FastDateFormat.getInstance(
@@ -130,7 +129,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     assert(castedTimestamp == expectedTime * 1000L)
 
     val customDate = "31/01/2015"
-    val dateOptions = new CSVOptions(Map("dateFormat" -> "dd/MM/yyyy"), false, "GMT")
+    val dateOptions = new CSVOptions(Map("dateFormat" -> "dd/MM/yyyy"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), dateOptions)
     format = FastDateFormat.getInstance(
       dateOptions.dateFormat,
@@ -139,7 +138,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     val expectedDate = format.parse(customDate).getTime
     val castedDate = parser.makeConverter("_1", DateType, nullable = true)
         .apply(customDate)
-    assert(castedDate == DateTimeUtils.millisToDays(expectedDate, ZoneOffset.UTC))
+    assert(castedDate == DateTimeUtils.millisToDays(expectedDate, UTC))
 
     val timestamp = "2015-01-01 00:00:00"
     timestampsOptions = new CSVOptions(Map(
@@ -154,7 +153,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Throws exception for casting an invalid string to Float and Double Types") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val types = Seq(DoubleType, FloatType)
     val input = Seq("10u000", "abc", "1 2/3")
@@ -169,7 +168,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Float NaN values are parsed correctly") {
-    val options = new CSVOptions(Map("nanValue" -> "nn"), false, "GMT")
+    val options = new CSVOptions(Map("nanValue" -> "nn"), false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val floatVal: Float = parser.makeConverter(
       "_1", FloatType, nullable = true).apply("nn").asInstanceOf[Float]
@@ -180,7 +179,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Double NaN values are parsed correctly") {
-    val options = new CSVOptions(Map("nanValue" -> "-"), false, "GMT")
+    val options = new CSVOptions(Map("nanValue" -> "-"), false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val doubleVal: Double = parser.makeConverter(
       "_1", DoubleType, nullable = true).apply("-").asInstanceOf[Double]
@@ -189,14 +188,14 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Float infinite values can be parsed") {
-    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "GMT")
+    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "UTC")
     var parser = new UnivocityParser(StructType(Seq.empty), negativeInfOptions)
     val floatVal1 = parser.makeConverter(
       "_1", FloatType, nullable = true).apply("max").asInstanceOf[Float]
 
     assert(floatVal1 == Float.NegativeInfinity)
 
-    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "GMT")
+    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), positiveInfOptions)
     val floatVal2 = parser.makeConverter(
       "_1", FloatType, nullable = true).apply("max").asInstanceOf[Float]
@@ -205,14 +204,14 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Double infinite values can be parsed") {
-    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "GMT")
+    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "UTC")
     var parser = new UnivocityParser(StructType(Seq.empty), negativeInfOptions)
     val doubleVal1 = parser.makeConverter(
       "_1", DoubleType, nullable = true).apply("max").asInstanceOf[Double]
 
     assert(doubleVal1 == Double.NegativeInfinity)
 
-    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "GMT")
+    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), positiveInfOptions)
     val doubleVal2 = parser.makeConverter(
       "_1", DoubleType, nullable = true).apply("max").asInstanceOf[Double]
@@ -228,7 +227,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
       val df = new DecimalFormat("", new DecimalFormatSymbols(Locale.forLanguageTag(langTag)))
       val input = df.format(expected.toBigDecimal)
 
-      val options = new CSVOptions(Map("locale" -> langTag), false, "GMT")
+      val options = new CSVOptions(Map("locale" -> langTag), false, "UTC")
       val parser = new UnivocityParser(new StructType().add("d", decimalType), options)
 
       assert(parser.makeConverter("_1", decimalType).apply(input) === expected)
@@ -263,7 +262,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     val input = "name\t42"
     val expected = UTF8String.fromString(input)
 
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
 
     val convertedValue = parser.makeConverter("_1", StringBasedUDT, nullable = false).apply(input)
@@ -280,7 +279,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
         filters: Seq[Filter],
         expected: Option[InternalRow]): Unit = {
       Seq(false, true).foreach { columnPruning =>
-        val options = new CSVOptions(Map.empty[String, String], columnPruning, "GMT")
+        val options = new CSVOptions(Map.empty[String, String], columnPruning, "UTC")
         val parser = new UnivocityParser(dataSchema, requiredSchema, options, filters)
         val actual = parser.parse(input)
         assert(actual === expected)
@@ -355,8 +354,8 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     check(new UnivocityParser(StructType(Seq.empty), options))
 
-    val optionsWithPattern =
-      new CSVOptions(Map("timestampFormat" -> "invalid", "dateFormat" -> "invalid"), false, "UTC")
+    val optionsWithPattern = new CSVOptions(
+      Map("timestampFormat" -> "invalid", "dateFormat" -> "invalid"), false, "UTC")
     check(new UnivocityParser(StructType(Seq.empty), optionsWithPattern))
   }
 }