Merge pull request #91 from pspoerri/ps/fix_sortshufflewriter

Fix GithubCI build and breaking change in SortShuffleWriter

Author: BernardMetzler

.github/workflows/ci.yml

@@ -17,55 +17,32 @@ jobs:
       fail-fast: true
       matrix:
         include:
-          - spark: 3.2.0
-            scala: 2.12.15
-          - spark: 3.2.0
-            scala: 2.13.5
-          - spark: 3.2.1
-            scala: 2.12.15
-          - spark: 3.2.1
-            scala: 2.13.5
-          - spark: 3.2.3
-            scala: 2.12.15
-          - spark: 3.2.3
-            scala: 2.13.5
-          - spark: 3.2.4
-            scala: 2.12.15
-          - spark: 3.2.4
-            scala: 2.13.5
-          - spark: 3.3.0
-            scala: 2.12.15
-          - spark: 3.3.0
-            scala: 2.13.8
-          - spark: 3.3.1
-            scala: 2.12.15
-          - spark: 3.3.1
-            scala: 2.13.8
-          - spark: 3.3.2
-            scala: 2.12.15
-          - spark: 3.3.2
-            scala: 2.13.8
-          - spark: 3.4.0
-            scala: 2.12.17
-          - spark: 3.4.0
-            scala: 2.13.8
-          - spark: 3.5.0
-            scala: 2.12.18
-          - spark: 3.5.0
-            scala: 2.13.8
-    runs-on: ubuntu-22.04 # Upgrading this version requires an additional sbt setup step.
+        - spark: 3.4.4
+          scala: 2.12.17
+        - spark: 3.4.4
+          scala: 2.13.8
+        - spark: 3.5.2
+          scala: 2.12.18
+        - spark: 3.5.2
+          scala: 2.13.8
+        - spark: 3.5.5
+          scala: 2.12.18
+        - spark: 3.5.5
+          scala: 2.13.8
+    runs-on: ubuntu-22.04
     env:
       SPARK_VERSION: ${{ matrix.spark }}
       SCALA_VERSION: ${{ matrix.scala }}
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
     - name: Setup JDK
-      uses: actions/setup-java@v3
+      # https://github.com/actions/setup-java?tab=readme-ov-file#caching-sbt-dependencies
+      uses: actions/setup-java@v4
       with:
-        distribution: temurin
-        java-version: 11
-        cache: sbt
+        distribution: 'temurin'
+        java-version: '17'
+        cache: 'sbt'
     - name: Check formatting
       shell: bash
       run: |-
@@ -74,13 +51,16 @@ jobs:
         sbt scalafmtCheckAll
     - name: Run tests
       shell: bash
+      env:
+        JAVA_OPTS: "--add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED  --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/javax.security.auth=ALL-UNNAMED"
       run: |
         sbt test
     - name: Run tests with Spark Shuffle Fetch enabled
       shell: bash
       if: ${{ !startsWith(matrix.spark, '3.2.') }}
       env:
         USE_SPARK_SHUFFLE_FETCH: "true"
+        JAVA_OPTS: "--add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED  --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/javax.security.auth=ALL-UNNAMED"
       run: |
         sbt test
     - name: Package

build.sbt

@@ -6,7 +6,7 @@
 scalaVersion := sys.env.getOrElse("SCALA_VERSION", "2.12.18")
 organization := "com.ibm"
 name := "spark-s3-shuffle"
-val sparkVersion = sys.env.getOrElse("SPARK_VERSION", "3.5.0")
+val sparkVersion = sys.env.getOrElse("SPARK_VERSION", "3.5.5")
 
 enablePlugins(GitVersioning, BuildInfoPlugin)
 

src/main/scala/org/apache/spark/shuffle/sort/S3ShuffleManager.scala

@@ -140,7 +140,7 @@ private[spark] class S3ShuffleManager(conf: SparkConf) extends ShuffleManager wi
           shuffleExecutorComponents
         )
       case other: BaseShuffleHandle[K @unchecked, V @unchecked, _] =>
-        new SortShuffleWriter(other, mapId, context, shuffleExecutorComponents)
+        new SortShuffleWriter(other, mapId, context, metrics, shuffleExecutorComponents)
     }
     new S3ShuffleWriter[K, V](writer)
   }