Try to fix some things

Author: allisonport-db

.github/workflows/iceberg_test.yaml

@@ -30,5 +30,6 @@ jobs:
 
       # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17.
       - name: Run unit tests (JDK 17)
+        # Disable coverage for now because it compiles all projects & causes a flink failure
         run: |
-          python run-tests.py --group kernel --coverage
+          python run-tests.py --group kernel

.github/workflows/kernel_test.yaml

@@ -48,6 +48,6 @@ jobs:
         # Thus, we need to publishM2 first so those jars are locally accessible.
         run: |
           build/sbt clean
-          build/sbt "++ $SCALA_VERSION publishM2"
+          build/sbt "++ $SCALA_VERSION sparkGroup/publishM2"
           cd examples/scala && build/sbt "++ $SCALA_VERSION compile"
         if: steps.git-diff.outputs.diff

.github/workflows/spark_examples_test.yaml

@@ -9,11 +9,11 @@ jobs:
         # These Scala versions must match those in the build.sbt
         scala: [2.13.13]
         # Important: This list of shards must be [0..NUM_SHARDS - 1]
-        shard: [0, 1, 2]
+        shard: [0, 1, 2, 3]
     env:
       SCALA_VERSION: ${{ matrix.scala }}
       # Important: This must be the same as the length of shards in matrix
-      NUM_SHARDS: 3
+      NUM_SHARDS: 4
     steps:
       - uses: actions/checkout@v3
       - uses: technote-space/get-diff-action@v4
@@ -81,11 +81,14 @@ jobs:
           pipenv run pip install numpy==1.21
           pipenv run pip install https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc4-bin/pyspark-4.0.0.tar.gz
         if: steps.git-diff.outputs.diff
-      - name: Run Spark Master tests
-        # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_test.yaml
-        # NOTE: in this branch, the default sparkVersion is the SPARK_MASTER_VERSION
+      - name: Run Delta Connect tests
         run: |
           TEST_PARALLELISM_COUNT=4 build/sbt -DsparkVersion=master "++ ${{ matrix.scala }}" clean connectServer/test
           TEST_PARALLELISM_COUNT=4 build/sbt -DsparkVersion=master "++ ${{ matrix.scala }}" clean connectServer/assembly connectClient/test
-          TEST_PARALLELISM_COUNT=4 pipenv run python run-tests.py --group spark
+        if: steps.git-diff.outputs.diff
+      - name: Run Delta Spark master tests
+        # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_test.yaml
+        # NOTE: in this branch, the default sparkVersion is the SPARK_MASTER_VERSION
+        run: |
+          TEST_PARALLELISM_COUNT=4 pipenv run python run-tests.py --group spark --shard ${{ matrix.shard }}
         if: steps.git-diff.outputs.diff

.github/workflows/spark_master_test.yaml

@@ -239,7 +239,7 @@ object TransactionSuite extends VectorTestUtils with MockEngineUtils {
         // Update the vectors
         val newColumnVectors = vectors.toBuffer
         newColumnVectors.remove(ordinal)
-        columnarBatch(newSchema, newColumnVectors)
+        columnarBatch(newSchema, newColumnVectors.toSeq)
       }
 
       override def getSize: Int = vectors.head.getSize

kernel/kernel-api/src/test/scala/io/delta/kernel/TransactionSuite.scala

@@ -257,7 +257,7 @@ class DeltaLogActionUtilsSuite extends AnyFunSuite with MockFileSystemClientUtil
     ).toInMemoryList.asScala
 
     assert(commitFiles.forall(fs => FileNames.isCommitFile(fs.getPath)))
-    assert(extractVersions(commitFiles) == Seq(10, 11, 12, 13, 14, 15, 16, 17))
+    assert(extractVersions(commitFiles.toSeq) == Seq(10, 11, 12, 13, 14, 15, 16, 17))
 
     val checkpointFiles = listDeltaLogFilesAsIter(
       createMockFSListFromEngine(checkpointsAndDeltas),
@@ -269,7 +269,7 @@ class DeltaLogActionUtilsSuite extends AnyFunSuite with MockFileSystemClientUtil
     ).toInMemoryList.asScala
 
     assert(checkpointFiles.forall(fs => FileNames.isCheckpointFile(fs.getPath)))
-    assert(extractVersions(checkpointFiles) == Seq(10, 14, 14, 17))
+    assert(extractVersions(checkpointFiles.toSeq) == Seq(10, 14, 14, 17))
   }
 
   test("listDeltaLogFiles: mustBeRecreatable") {