Upgrade Spark 4.0 dev1

Author: andresvelasquez2-affirm

assembly/pom.xml

@@ -146,6 +146,21 @@
       <artifactId>bcprov-jdk18on</artifactId>
       <scope>${hadoop.deps.scope}</scope>
     </dependency>
+
+    <!--
+      OpenLineage and Kafka dependencies for Affirm custom distribution.
+      These are bundled into /opt/spark/jars/ so all Spark applications have lineage tracking.
+    -->
+    <dependency>
+      <groupId>io.openlineage</groupId>
+      <artifactId>openlineage-spark_${scala.binary.version}</artifactId>
+      <version>${openlineage.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka-clients</artifactId>
+      <version>${kafka.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

pom.xml

@@ -102,10 +102,13 @@
     <module>examples</module>
     <module>repl</module>
     <module>launcher</module>
+    <!-- Disable Kafka connector modules because we are not using Spark with Kafka at Affirm -->
+    <!--
     <module>connector/kafka-0-10-token-provider</module>
     <module>connector/kafka-0-10</module>
     <module>connector/kafka-0-10-assembly</module>
     <module>connector/kafka-0-10-sql</module>
+    -->
     <module>connector/avro</module>
     <module>connector/protobuf</module>
     <!-- See additional modules enabled by profiles below -->
@@ -122,6 +125,7 @@
     <asm.version>9.7.1</asm.version>
     <slf4j.version>2.0.16</slf4j.version>
     <log4j.version>2.24.3</log4j.version>
+    <iceberg.version>1.10.1</iceberg.version>
     <!-- make sure to update IsolatedClientLoader whenever this version is changed -->
     <hadoop.version>3.4.1</hadoop.version>
     <!-- SPARK-41247: When updating `protobuf.version`, also need to update `protoVersion` in `SparkBuild.scala` -->
@@ -135,6 +139,8 @@
     <hive.version>2.3.10</hive.version>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
     <kafka.version>3.9.0</kafka.version>
+    <!-- OpenLineage for Spark lineage tracking - bundled in Affirm custom distribution -->
+    <openlineage.version>1.38.0</openlineage.version>
     <!-- After 10.17.1.0, the minimum required version is JDK19 -->
     <derby.version>10.16.1.1</derby.version>
     <parquet.version>1.15.2</parquet.version>
@@ -444,12 +450,53 @@
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter</artifactId>
       <scope>test</scope>
+    </dependency>
+      <!--
+        ADD THIS: AWS SDK v2 HTTP client for Hadoop 3.4 IRSA support
+
+        Hadoop 3.4 upgraded to AWS SDK v2 but doesn't include WebIdentityTokenFileCredentialsProvider
+        in the default S3A credential chain (fixed in Hadoop 3.5.0 via HADOOP-19535).
+
+        For IRSA (IAM Roles for Service Accounts) in EKS, we explicitly configure
+        WebIdentityTokenFileCredentialsProvider, which requires an HTTP client to make STS API calls.
+
+        This dependency provides the UrlConnectionHttpClient implementation needed by
+        WebIdentityTokenFileCredentialsProvider to assume roles via STS AssumeRoleWithWebIdentity.
+
+        Remove when upgrading to Hadoop 3.5+ or Spark 4.x with Hadoop 3.5+.
+      -->
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>url-connection-client</artifactId>
+      <version>2.29.52</version>
     </dependency>
     <dependency>
       <groupId>com.github.sbt.junit</groupId>
       <artifactId>jupiter-interface</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.mysql</groupId>
+      <artifactId>mysql-connector-j</artifactId>
+      <version>9.2.0</version>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.iceberg</groupId>
+      <artifactId>iceberg-spark-runtime-4.0_2.13</artifactId>
+      <version>${iceberg.version}</version>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.iceberg</groupId>
+      <artifactId>iceberg-aws-bundle</artifactId>
+      <version>${iceberg.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-aws</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
   </dependencies>
   <dependencyManagement>
     <dependencies>

python/.gitignore

@@ -5,5 +5,6 @@ pyspark_client.egg-info
 pyspark_connect.egg-info
 build/
 dist/
+venv/
 ./setup.py
 ./setup.cfg

python/MANIFEST.in

@@ -19,6 +19,7 @@
 recursive-include pyspark *.pyi py.typed *.json
 recursive-include deps/jars *.jar
 graft deps/bin
+graft deps/k8s
 recursive-include deps/sbin spark-config.sh spark-daemon.sh start-history-server.sh stop-history-server.sh
 recursive-include deps/data *.data *.txt
 recursive-include deps/licenses *.txt

python/packaging/classic/setup.py

@@ -105,13 +105,15 @@
 USER_SCRIPTS_PATH = os.path.join(SPARK_HOME, "sbin")
 DATA_PATH = os.path.join(SPARK_HOME, "data")
 LICENSES_PATH = os.path.join(SPARK_HOME, "licenses")
+DOCKER_PATH = os.path.join(SPARK_HOME, "resource-managers/kubernetes/docker/src/main/dockerfiles/spark")
 
 SCRIPTS_TARGET = os.path.join(TEMP_PATH, "bin")
 USER_SCRIPTS_TARGET = os.path.join(TEMP_PATH, "sbin")
 JARS_TARGET = os.path.join(TEMP_PATH, "jars")
 EXAMPLES_TARGET = os.path.join(TEMP_PATH, "examples")
 DATA_TARGET = os.path.join(TEMP_PATH, "data")
 LICENSES_TARGET = os.path.join(TEMP_PATH, "licenses")
+DOCKER_TARGET = os.path.join(TEMP_PATH, "k8s")
 
 # Check and see if we are under the spark path in which case we need to build the symlink farm.
 # This is important because we only want to build the symlink farm while under Spark otherwise we
@@ -221,6 +223,7 @@ def run(self):
             os.symlink(EXAMPLES_PATH, EXAMPLES_TARGET)
             os.symlink(DATA_PATH, DATA_TARGET)
             os.symlink(LICENSES_PATH, LICENSES_TARGET)
+            os.symlink(DOCKER_PATH, DOCKER_TARGET)
         else:
             # For windows fall back to the slower copytree
             copytree(JARS_PATH, JARS_TARGET)
@@ -229,6 +232,7 @@ def run(self):
             copytree(EXAMPLES_PATH, EXAMPLES_TARGET)
             copytree(DATA_PATH, DATA_TARGET)
             copytree(LICENSES_PATH, LICENSES_TARGET)
+            copytree(DOCKER_PATH, DOCKER_TARGET)
     else:
         # If we are not inside of SPARK_HOME verify we have the required symlink farm
         if not os.path.exists(JARS_TARGET):
@@ -296,6 +300,7 @@ def run(self):
             "pyspark.streaming",
             "pyspark.bin",
             "pyspark.sbin",
+            "pyspark.k8s",
             "pyspark.jars",
             "pyspark.pandas",
             "pyspark.pandas.data_type_ops",
@@ -321,6 +326,7 @@ def run(self):
             "pyspark.jars": "deps/jars",
             "pyspark.bin": "deps/bin",
             "pyspark.sbin": "deps/sbin",
+            "pyspark.k8s": "deps/k8s",
             "pyspark.python.lib": "lib",
             "pyspark.data": "deps/data",
             "pyspark.licenses": "deps/licenses",
@@ -329,6 +335,7 @@ def run(self):
         package_data={
             "pyspark.jars": ["*.jar"],
             "pyspark.bin": ["*"],
+            "pyspark.k8s": ["*"],
             "pyspark.sbin": [
                 "spark-config.sh",
                 "spark-daemon.sh",
@@ -398,11 +405,13 @@ def run(self):
             os.remove(os.path.join(TEMP_PATH, "examples"))
             os.remove(os.path.join(TEMP_PATH, "data"))
             os.remove(os.path.join(TEMP_PATH, "licenses"))
+            os.remove(os.path.join(TEMP_PATH, "k8s"))
         else:
             rmtree(os.path.join(TEMP_PATH, "jars"))
             rmtree(os.path.join(TEMP_PATH, "bin"))
             rmtree(os.path.join(TEMP_PATH, "sbin"))
             rmtree(os.path.join(TEMP_PATH, "examples"))
             rmtree(os.path.join(TEMP_PATH, "data"))
             rmtree(os.path.join(TEMP_PATH, "licenses"))
+            rmtree(os.path.join(TEMP_PATH, "k8s"))
         os.rmdir(TEMP_PATH)

python/pyspark/version.py

@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__: str = "4.0.0"
+__version__: str = "2815!4.0.0+affirm.dev1"

resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh

@@ -115,4 +115,5 @@ case "$1" in
 esac
 
 # Execute the container CMD under tini for better hygiene
-exec /usr/bin/tini -s -- "${CMD[@]}"
+export SPARK_VERSION="4.0.0"
+exec /tini -s -- "${CMD[@]}"