[Build] Pin Spark 4.2 lane to 4.2.0-preview5 (#6870)

## 🥞 Stacked PR
Use this [link](https://github.com/delta-io/delta/pull/6870/files) to
review incremental changes.
-
[**stack/spark42-preview**](#6870)
[[Files changed](https://github.com/delta-io/delta/pull/6870/files)]
- [stack/bump_uc_sha](#6866)
[[Files
changed](https://github.com/delta-io/delta/pull/6866/files/046318c601ad27e6edbb069fba558bff95b047e5..11a3e61ca79e90de45ee2252c2cefc06d6ec5ebb)]
-
[stack/DeltaCatalogClient_create](#6826)
[[Files
changed](https://github.com/delta-io/delta/pull/6826/files/11a3e61ca79e90de45ee2252c2cefc06d6ec5ebb..8a63fe4049beb49bf93f4d1b3a308b56ca5a273f)]
-
[stack/DeltaCatalogClient_replace](#6859)
[[Files
changed](https://github.com/delta-io/delta/pull/6859/files/8a63fe4049beb49bf93f4d1b3a308b56ca5a273f..0c5b785015d413cd9c026cd1091cf7f3d457f5f6)]

---------
#### Which Delta project/connector is this regarding?
- [x] Spark
- [ ] Standalone
- [ ] Flink
- [ ] Kernel
- [ ] Other (fill in here)

## Description
[Build] Pin Spark 4.2 lane to 4.2.0-preview5

- `CrossSparkVersions.scala`: `4.2.0-SNAPSHOT` -> `4.2.0-preview5`; drop
jitpack.
- `get_spark_version_info.py`: `--released-spark-versions` explicitly
excludes `-SNAPSHOT` / `-preview` markers.
- `test_cross_spark_publish.py`: bump pinned version; widen version
regex to accept `-previewN`; mirror the explicit pre-release marker
check.
- `spark_test.yaml`: comment "snapshot" -> "preview".
- `DeltaChangelogTestBase.java`: widen the existing SNAPSHOT-only skip
to also skip `-preview` (known `DeltaCatalog`/`AbstractDeltaCatalog`
classpath bug in sparkV2).
- `DeltaParquetFormatVersionSuite.scala`: widen the SPARK-56414
SNAPSHOT-only `assume` to also skip `-preview` (Delta's
`DeltaFileFormatWriter` does not yet merge per-write options).


## How was this patch tested?
CI

## Does this PR introduce _any_ user-facing changes?
No

Signed-off-by: Yi Li <yi.li@databricks.com>

Author: yili-db

.github/workflows/spark_test.yaml

@@ -37,8 +37,8 @@ jobs:
     name: "DS: Spark ${{ matrix.spark_version }}, Scala ${{ matrix.scala }}, Shard ${{ matrix.shard }}"
     runs-on: ubuntu-24.04
     needs: generate-matrix
-    # Spark 4.2 is a branch snapshot until the Spark 4.2 release is published.
-    # Do not block PRs on its failures while the snapshot lane is stabilizing.
+    # Spark 4.2 is a preview release until the Spark 4.2 release is published.
+    # Do not block PRs on its failures while the preview lane is stabilizing.
     continue-on-error: ${{ matrix.spark_version == '4.2' }}
     strategy:
       fail-fast: false

project/CrossSparkVersions.scala

@@ -286,18 +286,15 @@ object SparkVersionSpec {
     jacksonVersion = "2.18.2"
   )
 
-  private val spark42Snapshot = SparkVersionSpec(
-    fullVersion = "4.2.0-SNAPSHOT",
+  private val spark42Preview = SparkVersionSpec(
+    fullVersion = "4.2.0-preview5",
     targetJvm = "17",
     additionalSourceDir = Some("scala-shims/spark-4.2"),
     supportIceberg = false,
     supportHudi = false,
     antlr4Version = "4.13.1",
     additionalJavaOptions = java17TestSettings,
-    jacksonVersion = "2.18.2",
-    // Artifact updates in maven central for roaringbitmap stopped after 1.3.0.
-    // Spark branch-4.2 uses 1.5.3. Relevant Spark PR here https://github.com/apache/spark/pull/52892
-    additionalResolvers = Seq("jitpack" at "https://jitpack.io")
+    jacksonVersion = "2.18.2"
   )
 
   /** Default Spark version */
@@ -307,7 +304,7 @@ object SparkVersionSpec {
   val MASTER: Option[SparkVersionSpec] = None
 
   /** All supported Spark versions - internal use only */
-  val ALL_SPECS = Seq(spark40, spark41, spark42Snapshot)
+  val ALL_SPECS = Seq(spark40, spark41, spark42Preview)
 }
 
 /** See docs on top of this file */

project/scripts/get_spark_version_info.py

@@ -111,10 +111,12 @@ def main():
             print(json.dumps(matrix_versions))
 
         elif args.released_spark_versions:
-            # Only include released versions (no -SNAPSHOT in fullVersion)
+            # Only include released versions; explicitly exclude pre-release markers
+            # (`-SNAPSHOT`, `-previewN`).
+            pre_release_markers = ("-SNAPSHOT", "-preview")
             matrix_versions = []
             for v in versions:
-                if "-SNAPSHOT" not in v["fullVersion"]:
+                if not any(m in v["fullVersion"] for m in pre_release_markers):
                     matrix_versions.append(v["shortVersion"])
             print(json.dumps(matrix_versions))
 

project/tests/test_cross_spark_publish.py

@@ -108,7 +108,7 @@ def all_jars(self) -> List[str]:
 SPARK_VERSIONS: Dict[str, SparkVersionSpec] = {
     "4.0.1": SparkVersionSpec(suffix="_4.0", support_iceberg=True, support_hudi=True),
     "4.1.0": SparkVersionSpec(suffix="_4.1", support_iceberg=False, support_hudi=False),
-    "4.2.0-SNAPSHOT": SparkVersionSpec(suffix="_4.2", support_iceberg=False, support_hudi=False)
+    "4.2.0-preview5": SparkVersionSpec(suffix="_4.2", support_iceberg=False, support_hudi=False)
 }
 
 # The default Spark version
@@ -320,9 +320,9 @@ def validate_spark_versions(self) -> None:
             )
 
             # Parse output - each line is a Spark version
-            # Version format: X.Y.Z or X.Y.Z-SNAPSHOT
+            # Version format: X.Y.Z, X.Y.Z-SNAPSHOT, X.Y.Z-previewN
             import re
-            version_pattern = re.compile(r'^\d+\.\d+\.\d+(-SNAPSHOT)?$')
+            version_pattern = re.compile(r'^\d+\.\d+\.\d+(-(SNAPSHOT|preview\d+))?$')
 
             build_versions = set()
             for line in result.stdout.strip().split('\n'):
@@ -469,7 +469,7 @@ def test_all_spark_versions(self) -> bool:
             return False
 
     def test_released_spark_versions(self) -> bool:
-        """Test that --released-spark-versions excludes snapshots."""
+        """Test that --released-spark-versions excludes snapshots and pre-releases."""
         if not self.ensure_json_exists():
             return False
 
@@ -493,22 +493,30 @@ def test_released_spark_versions(self) -> bool:
                 print("  ✗ All entries must be strings")
                 return False
 
-            # Load JSON and verify snapshots are excluded
+            # Load JSON and verify pre-release versions are excluded
+            # (`-SNAPSHOT`, `-previewN`).
+            pre_release_markers = ("-SNAPSHOT", "-preview")
+
+            def is_pre_release(full_version: str) -> bool:
+                return any(m in full_version for m in pre_release_markers)
+
             with open(self.json_path, 'r') as f:
                 data = json.load(f)
 
-            expected_count = sum(1 for entry in data if "-SNAPSHOT" not in entry["fullVersion"])
+            expected_count = sum(1 for entry in data if not is_pre_release(entry["fullVersion"]))
             if len(released_versions) != expected_count:
                 print(f"  ✗ Expected {expected_count} released versions, got {len(released_versions)}")
                 return False
 
-            # Verify no snapshot versions included
+            # Verify no pre-release versions included (rough check via shortVersion lookup)
+            full_versions_by_short = {entry["shortVersion"]: entry["fullVersion"] for entry in data}
             for version in released_versions:
-                if "SNAPSHOT" in version.upper():
-                    print(f"  ✗ Released versions should not include snapshots: {version}")
+                full = full_versions_by_short.get(version, "")
+                if is_pre_release(full):
+                    print(f"  ✗ Released versions should not include pre-releases: {version} ({full})")
                     return False
 
-            print(f"  ✓ --released-spark-versions: {released_versions} (snapshots excluded)")
+            print(f"  ✓ --released-spark-versions: {released_versions} (snapshots and previews excluded)")
             return True
 
         except (subprocess.CalledProcessError, json.JSONDecodeError) as e:

spark/src/test/scala/org/apache/spark/sql/delta/DeltaParquetFormatVersionSuite.scala

@@ -119,10 +119,14 @@ class DeltaParquetFormatVersionSuite
    * Guard for tests that need SPARK-56414 behavior (per-write options overriding session conf
    * in Parquet writes). SPARK-56414 is merged into Spark 4.2, but Delta's DeltaFileFormatWriter
    * (a fork of FileFormatWriter) does not yet call mergeWriteOptionsIntoHadoopConf. Skip on
-   * SNAPSHOT until DeltaFileFormatWriter is updated.
+   * Spark pre-release builds (SNAPSHOT, preview) until DeltaFileFormatWriter is updated.
    */
   private def assumeSpark56414Available(): Unit = {
-    assume(spark.version >= "4.2" && !spark.version.contains("SNAPSHOT"),
+    val sparkVersion = spark.version
+    assume(
+      sparkVersion >= "4.2" &&
+        !sparkVersion.contains("SNAPSHOT") &&
+        !sparkVersion.contains("preview"),
       "DeltaFileFormatWriter does not yet merge per-write options into Hadoop conf (SPARK-56414)")
   }
 

spark/v2/src/test/scala-shims/spark-4.2/io/delta/spark/internal/v2/read/changelog/DeltaChangelogTestBase.java

@@ -40,9 +40,10 @@ public abstract class DeltaChangelogTestBase extends DeltaV2TestBase {
 
   @BeforeAll
   public static void setUpChangelogSparkAndEngine() {
+    String sparkVersion = org.apache.spark.package$.MODULE$.SPARK_VERSION();
     org.junit.jupiter.api.Assumptions.assumeFalse(
-        org.apache.spark.package$.MODULE$.SPARK_VERSION().contains("SNAPSHOT"),
-        "Changelog tests are temporarily skipped on Spark SNAPSHOT builds"
+        sparkVersion.contains("SNAPSHOT") || sparkVersion.contains("preview"),
+        "Changelog tests are temporarily skipped on Spark pre-release builds (SNAPSHOT/preview)"
             + " due to a DeltaCatalog/AbstractDeltaCatalog classpath issue in sparkV2");
     if (spark != null) {
       spark.stop();