Add OpenLineage to Spark 3.5.4

assembly/pom.xml

@@ -85,6 +85,21 @@
       <artifactId>guava</artifactId>
       <scope>${hadoop.deps.scope}</scope>
     </dependency>
+
+    <!--
+      OpenLineage and Kafka dependencies for Affirm custom distribution.
+      These are bundled into /opt/spark/jars/ so all Spark applications have lineage tracking.
+    -->
+    <dependency>
+      <groupId>io.openlineage</groupId>
+      <artifactId>openlineage-spark_${scala.binary.version}</artifactId>
+      <version>${openlineage.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka-clients</artifactId>
+      <version>${kafka.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

pom.xml

@@ -142,6 +142,8 @@
     <hive.version.short>2.3</hive.version.short>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
     <kafka.version>3.4.1</kafka.version>
+    <!-- OpenLineage for Spark lineage tracking - bundled in Affirm custom distribution -->
+    <openlineage.version>1.38.0</openlineage.version>
     <!-- After 10.15.1.3, the minimum required version is JDK9 -->
     <derby.version>10.14.2.0</derby.version>
     <parquet.version>1.13.1</parquet.version>

python/pyspark/version.py

@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__: str = "2815!3.5.4+affirm.4"
+__version__: str = "2815!3.5.4+affirm.5"