Branch 3.0 post-rc2 cherry picks (#2180)

Author: scottsand-db

.github/workflows/connectors_test.yaml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       matrix:
         # These Scala versions must match those in the build.sbt
-        scala: [2.13.5, 2.12.15]
+        scala: [2.13.8, 2.12.17]
     steps:
       - uses: actions/checkout@v2
       - name: install java

.github/workflows/kernel_test.yaml

@@ -4,7 +4,7 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     env:
-      SCALA_VERSION: 2.12.15
+      SCALA_VERSION: 2.12.17
     steps:
       - uses: actions/checkout@v3
       - name: install java

.github/workflows/spark_test.yaml

@@ -6,7 +6,7 @@ jobs:
     strategy:
       matrix:
         # These Scala versions must match those in the build.sbt
-        scala: [2.12.15, 2.13.5]
+        scala: [2.12.17, 2.13.8]
     env:
       SCALA_VERSION: ${{ matrix.scala }}
     steps:

Dockerfile

@@ -25,6 +25,7 @@ RUN apt-get install -y wget
 RUN apt-get install -y openjdk-8-jdk
 RUN apt-get install -y python3.8
 RUN apt-get install -y python3-pip
+RUN apt-get install -y git
 
 # Upgrade pip. This is needed to use prebuilt wheels for packages cffi (dep of cryptography) and
 # cryptography. Otherwise, building wheels for these packages fails.

benchmarks/build.sbt

@@ -15,7 +15,7 @@
  */
 
 name := "benchmarks"
-scalaVersion := "2.12.15"
+scalaVersion := "2.12.17"
 
 lazy val root = (project in file("."))
   .settings(

build.sbt

@@ -21,15 +21,15 @@ import Mima._
 import Unidoc._
 
 // Scala versions
-val scala212 = "2.12.15"
-val scala213 = "2.13.5"
+val scala212 = "2.12.17"
+val scala213 = "2.13.8"
 val all_scala_versions = Seq(scala212, scala213)
 
 // Due to how publishArtifact is determined for javaOnlyReleaseSettings, incl. storage
 // It was necessary to change default_scala_version to scala213 in build.sbt
 // to build the project with Scala 2.13 only
 // As a setting, it's possible to set it on command line easily
-// sbt 'set default_scala_version := 2.13.5' [commands]
+// sbt 'set default_scala_version := 2.13.8' [commands]
 // FIXME Why not use scalaVersion?
 val default_scala_version = settingKey[String]("Default Scala version")
 Global / default_scala_version := scala212

connectors/.github/workflows/test.yaml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        scala: [2.13.8, 2.12.8, 2.11.12]
+        scala: [2.13.8, 2.12.17, 2.11.12]
     steps:
       - uses: actions/checkout@v2
       - name: install java

connectors/examples/build.sbt

@@ -18,11 +18,11 @@ name := "examples"
 organization := "com.examples"
 organizationName := "examples"
 
-scalaVersion := "2.12.8"
+scalaVersion := "2.12.17"
 version := "0.1.0"
 
 lazy val commonSettings = Seq(
-  crossScalaVersions := Seq("2.13.8", "2.12.8", "2.11.12"),
+  crossScalaVersions := Seq("2.13.8", "2.12.17", "2.11.12"),
   resolvers += Resolver.mavenLocal,
   libraryDependencies ++= Seq(
     "io.delta" %% "delta-standalone" % getStandaloneVersion(),
@@ -47,14 +47,14 @@ lazy val extraMavenRepo = sys.env.get("EXTRA_MAVEN_REPO").toSeq.map { repo =>
 
 lazy val convertToDelta = (project in file("convert-to-delta")) settings (
   name := "convert",
-  scalaVersion := "2.12.8",
+  scalaVersion := "2.12.17",
   commonSettings,
   extraMavenRepo
 )
 
 lazy val helloWorld = (project in file("hello-world")) settings (
   name := "hello",
-  scalaVersion := "2.12.8",
+  scalaVersion := "2.12.17",
   commonSettings,
   extraMavenRepo
 )
@@ -63,7 +63,7 @@ val flinkVersion = "1.16.1"
 val flinkHadoopVersion = "3.1.0"
 lazy val flinkExample = (project in file("flink-example")) settings (
   name := "flink",
-  scalaVersion := "2.12.8",
+  scalaVersion := "2.12.17",
   commonSettings,
   extraMavenRepo,
   resolvers += Resolver.mavenLocal,

connectors/examples/run_examples.py

@@ -108,7 +108,7 @@ def __exit__(self, tpe, value, traceback):
        by running the following commands in the root connectors folder.
        
        build/sbt '++2.11.12 publishM2'
-       build/sbt '++2.12.8 publishM2'
+       build/sbt '++2.12.17 publishM2'
        build/sbt '++2.13.8 publishM2'
     """
 
@@ -146,5 +146,5 @@ def __exit__(self, tpe, value, traceback):
         run_maven_proj(path.join(root_dir, dir), className, args.version, args.maven_repo, "2.13")
 
         run_sbt_proj(root_dir, proj, className, args.version, args.maven_repo, "2.11.12")
-        run_sbt_proj(root_dir, proj, className, args.version, args.maven_repo, "2.12.8")
+        run_sbt_proj(root_dir, proj, className, args.version, args.maven_repo, "2.12.17")
         run_sbt_proj(root_dir, proj, className, args.version, args.maven_repo, "2.13.8")

examples/scala/build.sbt

@@ -18,7 +18,7 @@ name := "example"
 organization := "com.example"
 organizationName := "example"
 
-val scala212 = "2.12.15"
+val scala212 = "2.12.17"
 val scala213 = "2.13.8"
 val deltaVersion = "2.1.0"
 val icebergVersion = "1.4.0"

icebergShaded/generate_iceberg_jars.py

@@ -25,7 +25,7 @@
 from os import path
 
 iceberg_lib_dir_name = "lib"
-iceberg_src_dir_name = "iceberg_src"
+iceberg_src_dir_name = "iceberg_src" # this is a git dir
 iceberg_patches_dir_name = "iceberg_src_patches"
 
 iceberg_src_commit_hash = "ede085d0f7529f24acd0c81dd0a43f7bb969b763"
@@ -44,7 +44,7 @@
     "data/build/libs/iceberg-data-*.jar"
 ]
 
-iceberg_root_dir = path.abspath(path.dirname(__file__))
+iceberg_root_dir = path.abspath(path.dirname(__file__)) # this is NOT a git dir
 iceberg_src_dir = path.join(iceberg_root_dir, iceberg_src_dir_name)
 iceberg_patches_dir = path.join(iceberg_root_dir, iceberg_patches_dir_name)
 iceberg_lib_dir = path.join(iceberg_root_dir, iceberg_lib_dir_name)
@@ -65,25 +65,11 @@ def iceberg_jars_exists():
     return True
 
 
-def set_git_config_if_empty(config_key, default_value):
-    curr_val = None
-    try:
-        (_, curr_val, _) = run_cmd("git config --get user.%s" % config_key)
-        curr_val = curr_val.decode("utf-8")
-    except:
-        print("Error getting user.%s" % config_key)
-    if not curr_val:
-        run_cmd("git config user.%s \"%s\"" % (config_key, default_value))
-
-
 def prepare_iceberg_source():
     with WorkingDirectory(iceberg_root_dir):
         print(">>> Cloning Iceberg repo")
         shutil.rmtree(iceberg_src_dir_name, ignore_errors=True)
 
-        set_git_config_if_empty("email", "<>")
-        set_git_config_if_empty("name", "Anonymous")
-
         # We just want the shallowest, smallest iceberg clone. We will check out the commit later.
         run_cmd("git clone --depth 1 --branch %s https://github.com/apache/iceberg.git %s" %
                 (iceberg_src_branch, iceberg_src_dir_name))

project/plugins.sbt

@@ -30,7 +30,18 @@ addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.3")
 
 addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.15")
 
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.6")
+//Upgrade sbt-scoverage to 2.0.3+ because 2.0.0 is not compatible to Scala 2.12.17:
+//sbt.librarymanagement.ResolveException: Error downloading org.scoverage:scalac-scoverage-plugin_2.12.17:2.0.0
+
+//It caused a conflict issue:
+//[error] java.lang.RuntimeException: found version conflict(s) in library dependencies; some are suspected to be binary incompatible:
+//[error] 
+//[error] 	* org.scala-lang.modules:scala-xml_2.12:2.1.0 (early-semver) is selected over 1.0.6
+//[error] 	    +- org.scoverage:scalac-scoverage-reporter_2.12:2.0.7 (depends on 2.1.0)
+//[error] 	    +- org.scalariform:scalariform_2.12:0.2.0             (depends on 1.0.6)
+//The following fix the conflict:
+libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always % "test"
 
 addSbtPlugin("net.aichler" % "sbt-jupiter-interface" % "0.9.1")
 

run-tests.py

@@ -67,7 +67,7 @@ def run_sbt_tests(root_dir, test_group, coverage, scala_version=None):
         cmd += ["+ %s" % test_cmd]  # build/sbt ... "+ project/test" ...
     else:
         # when no scala version is specified, run test with only the specified scala version
-        cmd += ["++ %s" % scala_version, test_cmd]  # build/sbt ... "++ 2.13.5" "project/test" ...
+        cmd += ["++ %s" % scala_version, test_cmd]  # build/sbt ... "++ 2.13.8" "project/test" ...
 
     if is_running_spark_tests:
         cmd += ["unidoc"]