[3.2][Spark] Upgrade Spark dependency to 3.5.3 (#3717)

Author: vkorukanti

.github/workflows/spark_test.yaml

@@ -56,7 +56,7 @@ jobs:
           # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from
           # the`version.sbt` file.
           pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0
-          pipenv run pip install pyspark==3.5.2
+          pipenv run pip install pyspark==3.5.3
           pipenv run pip install flake8==3.5.0 pypandoc==1.3.3
           pipenv run pip install importlib_metadata==3.10.0
           pipenv run pip install mypy==0.982

Dockerfile

@@ -31,7 +31,7 @@ RUN apt-get install -y git
 # cryptography. Otherwise, building wheels for these packages fails.
 RUN pip3 install --upgrade pip
 
-RUN pip3 install pyspark==3.5.2
+RUN pip3 install pyspark==3.5.3
 
 RUN pip3 install mypy==0.982
 

benchmarks/build.sbt

@@ -20,7 +20,7 @@ scalaVersion := "2.12.18"
 lazy val root = (project in file("."))
   .settings(
     name := "benchmarks",
-    libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.5.2" % "provided",
+    libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.5.3" % "provided",
     libraryDependencies += "com.github.scopt" %% "scopt" % "4.0.1",
     libraryDependencies += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.13.1",
 

build.sbt

@@ -34,7 +34,7 @@ val all_scala_versions = Seq(scala212, scala213)
 val default_scala_version = settingKey[String]("Default Scala version")
 Global / default_scala_version := scala212
 
-val LATEST_RELEASED_SPARK_VERSION = "3.5.2"
+val LATEST_RELEASED_SPARK_VERSION = "3.5.3"
 val SPARK_MASTER_VERSION = "4.0.0-SNAPSHOT"
 val sparkVersion = settingKey[String]("Spark version")
 spark / sparkVersion := getSparkVersion()

docs/environment.yml

@@ -33,7 +33,7 @@ dependencies:
       - packaging==23.2
       - py4j==0.10.9.7
       - pygments==2.16.1
-      - pyspark==3.5.2
+      - pyspark==3.5.3
       - pytz==2023.3.post1
       - requests==2.31.0
       - six==1.16.0

examples/scala/build.sbt

@@ -44,7 +44,7 @@ def getMajorMinor(version: String): (Int, Int) = {
 }
 val lookupSparkVersion: PartialFunction[(Int, Int), String] = {
   // versions 3.0.0 and above
-  case (major, minor) if major >= 3 => "3.5.2"
+  case (major, minor) if major >= 3 => "3.5.3"
   // versions 2.4.x
   case (major, minor) if major == 2 && minor == 4 => "3.4.0"
   // versions 2.3.x

setup.py

@@ -65,7 +65,7 @@ def run(self):
         'delta': ['py.typed'],
     },
     install_requires=[
-        'pyspark>=3.5.2,<3.6.0',
+        'pyspark>=3.5.3,<3.6.0',
         'importlib_metadata>=1.0.0',
     ],
     python_requires='>=3.6',