From f4bb9377d8e7e7ca36df60be76e54138531166bd Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Mon, 27 Jan 2025 13:25:43 +0100 Subject: [PATCH 1/6] Update SBT --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 336cc0be0..d4dbe6063 100644 --- a/project/build.properties +++ b/project/build.properties @@ -11,4 +11,4 @@ # limitations under the License. # -sbt.version=1.5.0 +sbt.version=1.10.7 From 141e0eb678570cb44b4439b8b3a468e98401e919 Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Wed, 9 Apr 2025 20:22:23 +0200 Subject: [PATCH 2/6] Add Scalafix --- .scalafix.conf | 17 +++++++++++++++++ project/plugins.sbt | 2 ++ .../standalone/internal/DeltaSharedTable.scala | 4 ++-- .../internal/DeltaSharingCDCReader.scala | 7 ++++--- .../internal/DeltaSharingHistoryManager.scala | 3 ++- 5 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 .scalafix.conf diff --git a/.scalafix.conf b/.scalafix.conf new file mode 100644 index 000000000..d9c4af847 --- /dev/null +++ b/.scalafix.conf @@ -0,0 +1,17 @@ +# Example .scalafix.conf section +OrganizeImports { + groups = [ + "re:javax?\\..*" # Matches java and javax + "re:scala\\..*" # Matches scala + "*" # Matches everything else (effectively the 3rdParty group) + "re:io\\.delta\\.sharing\\..*" # Matches deltaSharing specifically + ] + # You will need to configure other OrganizeImports options based on your preferences + # For example: + removeUnused = false + # blankLines = Auto + importsOrder = SymbolsFirst + importSelectorsOrder = Keep + groupedImports = Keep # Or Merge, Keep + # ... other options ... +} diff --git a/project/plugins.sbt b/project/plugins.sbt index f6fc12ffc..68db59a24 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -36,3 +36,5 @@ addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.16") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.11.12" + +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.2") diff --git a/server/src/main/scala/io/delta/standalone/internal/DeltaSharedTable.scala b/server/src/main/scala/io/delta/standalone/internal/DeltaSharedTable.scala index 966161baf..bf6bc6faf 100644 --- a/server/src/main/scala/io/delta/standalone/internal/DeltaSharedTable.scala +++ b/server/src/main/scala/io/delta/standalone/internal/DeltaSharedTable.scala @@ -22,6 +22,8 @@ import java.nio.charset.StandardCharsets.UTF_8 import java.util.Base64 import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer +import scala.util.control.NonFatal import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem import com.google.common.hash.Hashing @@ -36,8 +38,6 @@ import org.apache.hadoop.fs.azure.NativeAzureFileSystem import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem import org.apache.hadoop.fs.s3a.S3AFileSystem import org.apache.spark.sql.types.{DataType, MetadataBuilder, StructType} -import scala.collection.mutable.ListBuffer -import scala.util.control.NonFatal import scalapb.{GeneratedMessage, GeneratedMessageCompanion} import io.delta.sharing.server.{model, DeltaSharedTableProtocol, DeltaSharingIllegalArgumentException, DeltaSharingUnsupportedOperationException, ErrorStrings, QueryResult} diff --git a/server/src/main/scala/io/delta/standalone/internal/DeltaSharingCDCReader.scala b/server/src/main/scala/io/delta/standalone/internal/DeltaSharingCDCReader.scala index 920d6da4c..3b10cb7f5 100644 --- a/server/src/main/scala/io/delta/standalone/internal/DeltaSharingCDCReader.scala +++ b/server/src/main/scala/io/delta/standalone/internal/DeltaSharingCDCReader.scala @@ -19,6 +19,10 @@ package io.delta.standalone.internal import java.sql.Timestamp +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer +import scala.util.control.NonFatal + import io.delta.standalone.DeltaLog import io.delta.standalone.internal.actions.{ Action, @@ -31,9 +35,6 @@ import io.delta.standalone.internal.actions.{ import io.delta.standalone.internal.exception.DeltaErrors import io.delta.standalone.internal.util.ConversionUtils import org.apache.hadoop.conf.Configuration -import scala.collection.JavaConverters._ -import scala.collection.mutable.ListBuffer -import scala.util.control.NonFatal /** * This is a special CDCReader that is optimized for delta sharing server usage. diff --git a/server/src/main/scala/io/delta/standalone/internal/DeltaSharingHistoryManager.scala b/server/src/main/scala/io/delta/standalone/internal/DeltaSharingHistoryManager.scala index e99ec4e15..58c9620c2 100644 --- a/server/src/main/scala/io/delta/standalone/internal/DeltaSharingHistoryManager.scala +++ b/server/src/main/scala/io/delta/standalone/internal/DeltaSharingHistoryManager.scala @@ -22,12 +22,13 @@ import java.time.OffsetDateTime import java.time.ZoneOffset import java.time.format.DateTimeFormatter +import scala.collection.JavaConverters._ + import io.delta.standalone.internal.actions.CommitMarker import io.delta.standalone.internal.util.FileNames import io.delta.storage.LogStore import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path -import scala.collection.JavaConverters._ object DeltaSharingHistoryManager { /** From dacbc6a8745342e6ed1b9d55436f6a504aad0c19 Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Mon, 27 Jan 2025 13:28:59 +0100 Subject: [PATCH 3/6] Update ScalaTest --- build.sbt | 4 +++- .../scala/io/delta/sharing/server/CloudFileSignerSuite.scala | 4 ++-- .../io/delta/sharing/server/DeltaSharingServiceSuite.scala | 5 +++-- .../io/delta/sharing/server/SharedTableManagerSuite.scala | 4 ++-- .../io/delta/sharing/server/config/ServerConfigSuite.scala | 4 ++-- .../io/delta/standalone/internal/ColumnRangeSuite.scala | 4 ++-- .../standalone/internal/JsonPredicateEvaluatorV2Suite.scala | 4 ++-- .../standalone/internal/JsonPredicateFilterUtilsSuite.scala | 4 ++-- .../io/delta/standalone/internal/JsonPredicateSuite.scala | 4 ++-- .../standalone/internal/PartitionFilterUtilsSuite.scala | 4 ++-- .../io/delta/standalone/internal/TimestampUtilsSuite.scala | 4 ++-- 11 files changed, 24 insertions(+), 21 deletions(-) diff --git a/build.sbt b/build.sbt index cd1b7b6ab..faa846e41 100644 --- a/build.sbt +++ b/build.sbt @@ -245,7 +245,9 @@ lazy val server = (project in file("server")) enablePlugins(JavaAppPackaging) se "org.slf4j" % "slf4j-simple" % "1.6.1", "net.sourceforge.argparse4j" % "argparse4j" % "0.9.0", - "org.scalatest" %% "scalatest" % "3.0.5" % "test", + "org.scalatest" %% "scalatest" % "3.2.19" % "test", + "dev.zio" %% "zio-test" % "2.0.19" % "test", + "dev.zio" %% "zio-test-sbt" % "2.0.19" % "test", "org.bouncycastle" % "bcprov-jdk15on" % "1.70" % "test", "org.bouncycastle" % "bcpkix-jdk15on" % "1.70" % "test" ), diff --git a/server/src/test/scala/io/delta/sharing/server/CloudFileSignerSuite.scala b/server/src/test/scala/io/delta/sharing/server/CloudFileSignerSuite.scala index def1c0cca..867daa23b 100644 --- a/server/src/test/scala/io/delta/sharing/server/CloudFileSignerSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/CloudFileSignerSuite.scala @@ -17,11 +17,11 @@ package io.delta.sharing.server import org.apache.hadoop.fs.Path -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import io.delta.sharing.server.common.GCSFileSigner -class CloudFileSignerSuite extends FunSuite { +class CloudFileSignerSuite extends AnyFunSuite { test("GCSFileSigner.getBucketAndObjectNames") { assert(GCSFileSigner.getBucketAndObjectNames(new Path("gs://delta-sharing-test/foo")) diff --git a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala index e034a62b0..e93014ece 100644 --- a/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/DeltaSharingServiceSuite.scala @@ -28,7 +28,8 @@ import scala.collection.mutable.ArrayBuffer import com.linecorp.armeria.server.Server import io.delta.standalone.internal.DeltaSharedTable.{RESPONSE_FORMAT_DELTA, RESPONSE_FORMAT_PARQUET} import org.apache.commons.io.IOUtils -import org.scalatest.{BeforeAndAfterAll, FunSuite} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite import scalapb.json4s.JsonFormat import io.delta.sharing.server.DeltaSharingService.DELTA_SHARING_INCLUDE_END_STREAM_ACTION @@ -39,7 +40,7 @@ import io.delta.sharing.server.model._ import io.delta.sharing.server.protocol._ // scalastyle:off maxLineLength -class DeltaSharingServiceSuite extends FunSuite with BeforeAndAfterAll { +class DeltaSharingServiceSuite extends AnyFunSuite with BeforeAndAfterAll { def shouldRunIntegrationTest: Boolean = { sys.env.get("AWS_ACCESS_KEY_ID").exists(_.length > 0) && diff --git a/server/src/test/scala/io/delta/sharing/server/SharedTableManagerSuite.scala b/server/src/test/scala/io/delta/sharing/server/SharedTableManagerSuite.scala index 533645414..bd2ebc0fd 100644 --- a/server/src/test/scala/io/delta/sharing/server/SharedTableManagerSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/SharedTableManagerSuite.scala @@ -21,12 +21,12 @@ import java.util.{Arrays, Collections} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import io.delta.sharing.server.config.{SchemaConfig, ServerConfig, ShareConfig, TableConfig} import io.delta.sharing.server.protocol.{Schema, Share, Table} -class SharedTableManagerSuite extends FunSuite { +class SharedTableManagerSuite extends AnyFunSuite { test("list shares") { val serverConfig = new ServerConfig() diff --git a/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala b/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala index 5eae5cacb..af65241fa 100644 --- a/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala +++ b/server/src/test/scala/io/delta/sharing/server/config/ServerConfigSuite.scala @@ -22,9 +22,9 @@ import java.nio.file.Files import java.util.Arrays import org.apache.commons.io.FileUtils -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ServerConfigSuite extends FunSuite { +class ServerConfigSuite extends AnyFunSuite { def testConfig(content: String, serverConfig: ServerConfig): Unit = { val tempFile = Files.createTempFile("delta-sharing-server", ".yaml").toFile diff --git a/server/src/test/scala/io/delta/standalone/internal/ColumnRangeSuite.scala b/server/src/test/scala/io/delta/standalone/internal/ColumnRangeSuite.scala index 5a2908594..c28958659 100644 --- a/server/src/test/scala/io/delta/standalone/internal/ColumnRangeSuite.scala +++ b/server/src/test/scala/io/delta/standalone/internal/ColumnRangeSuite.scala @@ -18,11 +18,11 @@ package io.delta.standalone.internal -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import io.delta.sharing.server.common.TimestampUtils -class ColumnRangeSuite extends FunSuite { +class ColumnRangeSuite extends AnyFunSuite { test("invalid column range test") { assert(intercept[IllegalArgumentException] { diff --git a/server/src/test/scala/io/delta/standalone/internal/JsonPredicateEvaluatorV2Suite.scala b/server/src/test/scala/io/delta/standalone/internal/JsonPredicateEvaluatorV2Suite.scala index 9780eba09..5e9464486 100644 --- a/server/src/test/scala/io/delta/standalone/internal/JsonPredicateEvaluatorV2Suite.scala +++ b/server/src/test/scala/io/delta/standalone/internal/JsonPredicateEvaluatorV2Suite.scala @@ -18,11 +18,11 @@ package io.delta.standalone.internal -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import io.delta.sharing.server.common.{AndOp, BaseOp, ColumnOp, EqualOp, EvalContext, GreaterThanOp, GreaterThanOrEqualOp, IsNullOp, LessThanOp, LessThanOrEqualOp, LiteralOp, NotOp, OrOp} -class JsonPredicateEvaluatorV2Suite extends FunSuite { +class JsonPredicateEvaluatorV2Suite extends AnyFunSuite { // A wrapper around op evaluation in the specified context (data file). // diff --git a/server/src/test/scala/io/delta/standalone/internal/JsonPredicateFilterUtilsSuite.scala b/server/src/test/scala/io/delta/standalone/internal/JsonPredicateFilterUtilsSuite.scala index cf07abed3..e51e899b0 100644 --- a/server/src/test/scala/io/delta/standalone/internal/JsonPredicateFilterUtilsSuite.scala +++ b/server/src/test/scala/io/delta/standalone/internal/JsonPredicateFilterUtilsSuite.scala @@ -17,9 +17,9 @@ package io.delta.standalone.internal import io.delta.standalone.internal.actions.AddFile -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class JsonPredicateFilterUtilsSuite extends FunSuite { +class JsonPredicateFilterUtilsSuite extends AnyFunSuite { import JsonPredicateFilterUtils._ diff --git a/server/src/test/scala/io/delta/standalone/internal/JsonPredicateSuite.scala b/server/src/test/scala/io/delta/standalone/internal/JsonPredicateSuite.scala index ea95f592e..d6be342a6 100644 --- a/server/src/test/scala/io/delta/standalone/internal/JsonPredicateSuite.scala +++ b/server/src/test/scala/io/delta/standalone/internal/JsonPredicateSuite.scala @@ -18,11 +18,11 @@ package io.delta.standalone.internal -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import io.delta.sharing.server.common.{AndOp, BaseOp, ColumnOp, EqualOp, EvalContext, GreaterThanOp, IsNullOp, JsonUtils, LessThanOp, LiteralOp, NonLeafOp, NotOp, OrOp} -class JsonPredicateSuite extends FunSuite { +class JsonPredicateSuite extends AnyFunSuite { /** * A wrapper around op evaluation. diff --git a/server/src/test/scala/io/delta/standalone/internal/PartitionFilterUtilsSuite.scala b/server/src/test/scala/io/delta/standalone/internal/PartitionFilterUtilsSuite.scala index bd3399b9d..f5b657695 100644 --- a/server/src/test/scala/io/delta/standalone/internal/PartitionFilterUtilsSuite.scala +++ b/server/src/test/scala/io/delta/standalone/internal/PartitionFilterUtilsSuite.scala @@ -18,9 +18,9 @@ package io.delta.standalone.internal import io.delta.standalone.internal.actions.AddFile import org.apache.spark.sql.types.{IntegerType, StructField, StructType} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class PartitionFilterUtilsSuite extends FunSuite { +class PartitionFilterUtilsSuite extends AnyFunSuite { import PartitionFilterUtils._ diff --git a/server/src/test/scala/io/delta/standalone/internal/TimestampUtilsSuite.scala b/server/src/test/scala/io/delta/standalone/internal/TimestampUtilsSuite.scala index 2704f1549..cbc30be48 100644 --- a/server/src/test/scala/io/delta/standalone/internal/TimestampUtilsSuite.scala +++ b/server/src/test/scala/io/delta/standalone/internal/TimestampUtilsSuite.scala @@ -18,11 +18,11 @@ package io.delta.standalone.internal -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import io.delta.sharing.server.common.TimestampUtils -class TimestampUtilsSuite extends FunSuite { +class TimestampUtilsSuite extends AnyFunSuite { test("basic test") { // Only ISO 8601 is supported. TimestampUtils.parse("2023-06-10T00:00:00.000Z") From 5fc3fce8f17ebab68b9cb5b92ba560ffefaf02a9 Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Mon, 27 Jan 2025 13:34:45 +0100 Subject: [PATCH 4/6] Update Scala From 9f185a6686db7bb06fea3bf84d9666e4729a309a Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Mon, 27 Jan 2025 12:49:23 +0100 Subject: [PATCH 5/6] Update Armeria --- build.sbt | 2 +- .../io/delta/sharing/server/DeltaSharingService.scala | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index faa846e41..7f1d9728a 100644 --- a/build.sbt +++ b/build.sbt @@ -173,7 +173,7 @@ lazy val server = (project in file("server")) enablePlugins(JavaAppPackaging) se ExclusionRule("com.fasterxml.jackson.core"), ExclusionRule("com.fasterxml.jackson.module") ), - "com.linecorp.armeria" %% "armeria-scalapb" % "1.6.0" excludeAll( + "com.linecorp.armeria" %% "armeria-scalapb" % "1.9.2" excludeAll( ExclusionRule("com.fasterxml.jackson.core"), ExclusionRule("com.fasterxml.jackson.module"), ExclusionRule("org.json4s") diff --git a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala index 9fb55d16b..62408a0ee 100644 --- a/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala +++ b/server/src/main/scala/io/delta/sharing/server/DeltaSharingService.scala @@ -17,6 +17,7 @@ package io.delta.sharing.server import java.io.{ByteArrayOutputStream, File, FileNotFoundException} +import java.lang.reflect.{Field, Modifier} import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.AccessDeniedException import java.security.MessageDigest @@ -678,6 +679,15 @@ object DeltaSharingService { val defaultJsonPrinterField = Class.forName("com.linecorp.armeria.server.scalapb.ScalaPbConverterUtil$") .getDeclaredField("defaultJsonPrinter") + + // Remove the final modifier + val modifiersField = classOf[Field].getDeclaredField("modifiers") + modifiersField.setAccessible(true) + modifiersField.setInt( + defaultJsonPrinterField, + defaultJsonPrinterField.getModifiers & ~Modifier.FINAL + ) + defaultJsonPrinterField.setAccessible(true) defaultJsonPrinterField.set(module, new Printer()) } From c46de1f8edac1421645e2883eb441c51d9b11d3f Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Mon, 27 Jan 2025 18:11:02 +0100 Subject: [PATCH 6/6] Update Jackson --- build.sbt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 7f1d9728a..24d40afe4 100644 --- a/build.sbt +++ b/build.sbt @@ -165,10 +165,10 @@ lazy val server = (project in file("server")) enablePlugins(JavaAppPackaging) se libraryDependencies ++= Seq( // Pin versions for jackson libraries as the new version of `jackson-module-scala` introduces a // breaking change making us not able to use `delta-standalone`. - "com.fasterxml.jackson.core" % "jackson-core" % "2.6.7", - "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.7.3", - "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.6.7.1", - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % "2.6.7", + "com.fasterxml.jackson.core" % "jackson-core" % "2.15.2", + "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.2", + "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.2", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % "2.15.2", "org.json4s" %% "json4s-jackson" % "3.5.3" excludeAll( ExclusionRule("com.fasterxml.jackson.core"), ExclusionRule("com.fasterxml.jackson.module")