update relocated streaming imports and override slf4j 1.7.6 pinning

Signed-off-by: Sotaro Hikita <bering1814@gmail.com>

Author: lawofcycles

spark/sql-40/build.gradle

@@ -26,6 +26,15 @@ sparkVariants {
             }
         }
 
+        // Override BuildPlugin's slf4j 1.7.6 pinning: Spark 4.1+ requires slf4j 2.x
+        project.configurations.all { Configuration conf ->
+            conf.resolutionStrategy.eachDependency { details ->
+                if (details.requested.group == 'org.slf4j' && details.requested.name == 'slf4j-api') {
+                    details.useVersion '2.0.16'
+                }
+            }
+        }
+
         // Configure main compile task
         project.getTasks().getByName(scalaCompileTaskName) { ScalaCompile compileScala ->
             configure(compileScala.scalaCompileOptions.forkOptions) {

spark/sql-40/src/main/scala/org/opensearch/spark/sql/streaming/NullMetadataLog.scala

@@ -29,7 +29,7 @@
 
 package org.opensearch.spark.sql.streaming
 
-import org.apache.spark.sql.execution.streaming.MetadataLog
+import org.apache.spark.sql.execution.streaming.checkpointing.MetadataLog
 
 /**
  * A null object style metadata log that discards incoming data, and otherwise

spark/sql-40/src/main/scala/org/opensearch/spark/sql/streaming/OpenSearchCommitProtocol.scala

@@ -31,7 +31,7 @@ package org.opensearch.spark.sql.streaming
 
 import org.apache.commons.logging.Log
 import org.apache.commons.logging.LogFactory
-import org.apache.spark.sql.execution.streaming.MetadataLog
+import org.apache.spark.sql.execution.streaming.checkpointing.MetadataLog
 import org.opensearch.hadoop.util.Assert
 
 /**

spark/sql-40/src/main/scala/org/opensearch/spark/sql/streaming/OpenSearchSinkMetadataLog.scala

@@ -30,7 +30,7 @@
 package org.opensearch.spark.sql.streaming
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.execution.streaming.CompactibleFileStreamLog
+import org.apache.spark.sql.execution.streaming.runtime.CompactibleFileStreamLog
 import org.opensearch.hadoop.cfg.Settings
 
 /**

spark/sql-40/src/main/scala/org/opensearch/spark/sql/streaming/OpenSearchSparkSqlStreamingSink.scala

@@ -38,7 +38,7 @@ import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.SQLExecution
-import org.apache.spark.sql.execution.streaming.MetadataLog
+import org.apache.spark.sql.execution.streaming.checkpointing.MetadataLog
 import org.apache.spark.sql.execution.streaming.Sink
 import org.opensearch.hadoop.cfg.Settings