Bump UC sha to af090e7 and use the new UC-Spark package name

Signed-off-by: Yi Li <yi.li@databricks.com>

Author: yili-db

.github/actions/setup-unitycatalog/action.yml

@@ -2,7 +2,11 @@ name: "Set up pinned Unity Catalog build"
 description: >-
   Publishes Unity Catalog jars from the commit pinned in project/scripts/setup_unitycatalog_main.sh
   (the UC_PIN_SHA= line) to the runner's local Ivy / Maven caches, using GitHub Actions cache so the
-  slow UC build only runs the first time a pin is seen.
+  slow UC build only runs the first time a pin is seen. Reads SPARK_VERSION (Spark major.minor
+  short form) from the calling job's env to pick the Spark variant UC builds against; matrix
+  workflows set this from `matrix.spark_version`, other workflows leave it unset and inherit the
+  script's default. The cache key reflects SPARK_VERSION literally - workflows that share the
+  default share one cache entry under an empty `spark-` segment.
 
 runs:
   using: "composite"
@@ -15,9 +19,9 @@ runs:
         path: |
           ~/.ivy2/local
           ~/.m2/repository/io/unitycatalog
-        # Cache key hashes the setup script, so bumping UC_PIN_SHA (or any other script change)
-        # invalidates the cache.
-        key: uc-jars-${{ runner.os }}-${{ hashFiles('project/scripts/setup_unitycatalog_main.sh') }}
+        # Cache key hashes the setup script (so any script change invalidates) and includes the
+        # Spark short version (so 4.0 and 4.1 don't fight over a single shared cache entry).
+        key: uc-jars-${{ runner.os }}-spark${{ env.SPARK_VERSION }}-${{ hashFiles('project/scripts/setup_unitycatalog_main.sh') }}
     - name: Build Unity Catalog from pinned SHA
       shell: bash
       run: bash project/scripts/setup_unitycatalog_main.sh
@@ -33,4 +37,4 @@ runs:
         path: |
           ~/.ivy2/local
           ~/.m2/repository/io/unitycatalog
-        key: uc-jars-${{ runner.os }}-${{ hashFiles('project/scripts/setup_unitycatalog_main.sh') }}
+        key: uc-jars-${{ runner.os }}-spark${{ env.SPARK_VERSION }}-${{ hashFiles('project/scripts/setup_unitycatalog_main.sh') }}

.github/workflows/build.yaml

@@ -38,10 +38,20 @@ jobs:
           key: delta-sbt-cache-cross-spark
 
       # publishM2 compiles every aggregated project, including storage, which has
-      # unitycatalog-client as a compile-scope dependency. Publish the pinned UC build locally
-      # first so Delta compiles against the UC APIs it actually targets.
-      - name: Set up pinned Unity Catalog
+      # unitycatalog-client as a compile-scope dependency. test_cross_spark_publish.py also
+      # iterates over released Spark versions (sbt -DsparkVersion=<X.Y>), so we need UC's
+      # spark connector published for each variant Delta will resolve. Invoke the setup action
+      # once per Spark variant; the action's cache key partitions by SPARK_VERSION so each
+      # variant warms its own cache entry. Keep this list in sync with the released versions
+      # in project/spark-versions.json (snapshot versions are skipped by the cross-Spark test).
+      - name: Set up pinned Unity Catalog (Spark 4.0)
         uses: ./.github/actions/setup-unitycatalog
+        env:
+          SPARK_VERSION: "4.0"
+      - name: Set up pinned Unity Catalog (Spark 4.1)
+        uses: ./.github/actions/setup-unitycatalog
+        env:
+          SPARK_VERSION: "4.1"
 
       - name: Run cross-Spark build test
         run: python project/tests/test_cross_spark_publish.py

.github/workflows/spark_examples_test.yaml

@@ -45,6 +45,10 @@ jobs:
         scala: [2.13.17]
     env:
       SCALA_VERSION: ${{ matrix.scala }}
+      # Short Spark version (e.g. "4.0"). The UC setup action reads this from job env to pick the
+      # variant of UC to build. Per-step `env:` blocks below override this with the full version
+      # for Delta and examples sbt invocations that want "4.0.1"-style values.
+      SPARK_VERSION: ${{ matrix.spark_version }}
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Get Spark version details

build.sbt

@@ -786,6 +786,12 @@ val unityCatalogVersion: String = sys.props.getOrElse(
   if (useDefaultUnityCatalogReleaseVersion) defaultUnityCatalogReleaseVersion
   else unityCatalogReleaseVersion.getOrElse(pinnedUnityCatalogVersion))
 
+// UC publishes its Spark connector per Spark major.minor (e.g. unitycatalog-spark_4.1). This
+// is the artifact name without the Scala suffix - sbt's `%%` appends `_2.13` for dep
+// resolution; the canary check below appends `_2.13` explicitly for the Ivy/Maven path.
+val unityCatalogSparkArtifactName: String =
+  s"unitycatalog-spark_${CrossSparkVersions.getSparkVersionSpec().shortVersion}"
+
 /**
  * Returns true when `current` is at least `target`. Numeric segments only; suffix after
  * the first `-` (e.g. `-SNAPSHOT-abc1234`) is stripped before comparison.
@@ -831,7 +837,12 @@ def publishPinnedUnityCatalog(log: sbt.util.Logger, canary: java.io.File): Unit
   val procLogger = ProcessLogger(
     line => log.info(s"[UC setup] $line"),
     line => log.warn(s"[UC setup] $line"))
-  val exit = Process(Seq("bash", unityCatalogSetupScript)).!(procLogger)
+  // SPARK_VERSION tells the script which Spark variant to build (forwarded to UC's sbt as
+  // -DsparkVersion).
+  val exit = Process(
+    Seq("bash", unityCatalogSetupScript),
+    None,
+    "SPARK_VERSION" -> CrossSparkVersions.getSparkVersionSpec().shortVersion).!(procLogger)
   if (exit != 0) {
     sys.error(
       s"[UC] $unityCatalogSetupScript exited with code $exit. Run it manually to see full output.")
@@ -853,13 +864,17 @@ Global / ensurePinnedUnityCatalog := {
     sys.props.contains("unityCatalogVersion")
   if (unityCatalogReleaseVersion.isEmpty && !usingReleasedVersion) {
     val home = file(sys.props("user.home"))
+    // Canary on the spark artifact, not client/server: those are Spark-version-independent and
+    // would short-circuit the trigger when only the active Spark version changed, leaving the
+    // needed unitycatalog-spark_${X.Y}_2.13 unpublished.
+    val sparkArtifact = s"${unityCatalogSparkArtifactName}_2.13"
     // Check both layouts: a restored sbt cache can pre-populate ivy alone, leaving m2 empty -
     // checking only ivy would silently skip the slow publish and break mvn-based consumers.
     val ivy2Canary = home / ".ivy2" / "local" / "io.unitycatalog" /
-      "unitycatalog-client" / unityCatalogVersion / "ivys" / "ivy.xml"
+      sparkArtifact / unityCatalogVersion / "ivys" / "ivy.xml"
     val m2Canary = home / ".m2" / "repository" / "io" / "unitycatalog" /
-      "unitycatalog-client" / unityCatalogVersion /
-      s"unitycatalog-client-$unityCatalogVersion.pom"
+      sparkArtifact / unityCatalogVersion /
+      s"$sparkArtifact-$unityCatalogVersion.pom"
     if (!ivy2Canary.exists || !m2Canary.exists) {
       publishPinnedUnityCatalog(log, ivy2Canary)
     }
@@ -913,7 +928,7 @@ lazy val sparkUnityCatalog = (project in file("spark/unitycatalog"))
       "org.projectlombok" % "lombok" % "1.18.34" % "test",
 
       // Unity Catalog dependencies - exclude Jackson to use Spark's Jackson 2.15.x
-      "io.unitycatalog" %% "unitycatalog-spark" % unityCatalogVersion % "test" excludeAll(
+      "io.unitycatalog" %% unityCatalogSparkArtifactName % unityCatalogVersion % "test" excludeAll(
         ExclusionRule(organization = "com.fasterxml.jackson.core"),
         ExclusionRule(organization = "com.fasterxml.jackson.module"),
         ExclusionRule(organization = "com.fasterxml.jackson.datatype"),

project/scripts/setup_unitycatalog_main.sh

@@ -42,10 +42,17 @@
 #   4. Open a focused PR.
 #
 # Environment overrides:
-#   UC_DIR   directory to clone into       (default: /tmp/unitycatalog)
-#   UC_REPO  git remote URL                (default: upstream unitycatalog)
-#   UC_REF   must be `main` or UC_PIN_SHA  (default: UC_PIN_SHA below)
-#   UC_FORCE set to "1" to rebuild even when the Ivy artifact exists
+#   UC_DIR        directory to clone into                 (default: /tmp/unitycatalog)
+#   UC_REPO       git remote URL                          (default: upstream unitycatalog)
+#   UC_REF        must be `main` or UC_PIN_SHA            (default: UC_PIN_SHA below)
+#   UC_FORCE      set to "1" to rebuild even when the Ivy artifact exists
+#   SPARK_VERSION Spark major.minor UC should build for   (default: 4.1)
+#                 Forwarded as -DsparkVersion to UC's sbt; also determines the published artifact
+#                 name (unitycatalog-spark_${X.Y}_2.13). Delta's build.sbt sets this from
+#                 CrossSparkVersions when invoking the script; matrix CI workflows set it from
+#                 `matrix.spark_version`. Default matches UC's own default in
+#                 project/spark-versions.json - workflows that don't care which Spark variant UC
+#                 builds (kernel/flink/etc.) inherit it.
 #
 # UC_REF is restricted to exactly two values by design: the pinned SHA (the normal case) or
 # `main` (for the floating-main canary flow). Any other value is rejected. CI should never set
@@ -57,14 +64,15 @@ set -euo pipefail
 # The pin. Bump both lines together if UC's version.sbt changed at the new SHA. build.sbt's
 # `unityCatalogVersion` is obtained by running this script with `--print-version`, so these two
 # values are the single source of truth.
-UC_PIN_SHA=cd6fec685e9c2ce23fd85552ffac94c7b7e9cbd6
+UC_PIN_SHA=af090e73979bc216a0fe8feff59a5bbce0f41f14
 UC_BASE_VERSION=0.5.0-SNAPSHOT
 # ---------------------------------------------------------------------------------------------
 
 UC_DIR="${UC_DIR:-/tmp/unitycatalog}"
 UC_REPO="${UC_REPO:-https://github.com/unitycatalog/unitycatalog.git}"
 UC_REF="${UC_REF:-$UC_PIN_SHA}"
 UC_FORCE="${UC_FORCE:-0}"
+SPARK_VERSION="${SPARK_VERSION:-4.1}"
 
 # Enforce the two-value contract. Anything else is either a typo or a misuse and would bypass the
 # safety check below.
@@ -92,15 +100,19 @@ fi
 # Canonical Ivy + Maven artifact paths. Delta depends on all three UC modules; sbt resolves from
 # ~/.ivy2/local, mvn (kernel-examples integration tests) resolves from ~/.m2/repository. If any
 # is missing in either layout we must re-publish.
+# UC publishes its Spark connector under a per-Spark-version coordinate
+# (e.g. unitycatalog-spark_4.1_2.13). The suffix tracks SPARK_VERSION so the
+# canary check matches whatever variant we tell UC to build below.
+UC_SPARK_ARTIFACT="unitycatalog-spark_${SPARK_VERSION}_2.13"
 IVY_LOCAL="$HOME/.ivy2/local/io.unitycatalog"
 IVY_CANARY_CLIENT="$IVY_LOCAL/unitycatalog-client/$UC_VERSION/ivys/ivy.xml"
 IVY_CANARY_SERVER="$IVY_LOCAL/unitycatalog-server/$UC_VERSION/ivys/ivy.xml"
-IVY_CANARY_SPARK="$IVY_LOCAL/unitycatalog-spark_2.13/$UC_VERSION/ivys/ivy.xml"
+IVY_CANARY_SPARK="$IVY_LOCAL/$UC_SPARK_ARTIFACT/$UC_VERSION/ivys/ivy.xml"
 IVY_CANARY_HADOOP="$IVY_LOCAL/unitycatalog-hadoop/$UC_VERSION/ivys/ivy.xml"
 M2_LOCAL="$HOME/.m2/repository/io/unitycatalog"
 M2_CANARY_CLIENT="$M2_LOCAL/unitycatalog-client/$UC_VERSION/unitycatalog-client-$UC_VERSION.pom"
 M2_CANARY_SERVER="$M2_LOCAL/unitycatalog-server/$UC_VERSION/unitycatalog-server-$UC_VERSION.pom"
-M2_CANARY_SPARK="$M2_LOCAL/unitycatalog-spark_2.13/$UC_VERSION/unitycatalog-spark_2.13-$UC_VERSION.pom"
+M2_CANARY_SPARK="$M2_LOCAL/$UC_SPARK_ARTIFACT/$UC_VERSION/$UC_SPARK_ARTIFACT-$UC_VERSION.pom"
 M2_CANARY_HADOOP="$M2_LOCAL/unitycatalog-hadoop/$UC_VERSION/unitycatalog-hadoop-$UC_VERSION.pom"
 ALL_CANARIES=("$IVY_CANARY_CLIENT" "$IVY_CANARY_SERVER" "$IVY_CANARY_SPARK" "$IVY_CANARY_HADOOP"
               "$M2_CANARY_CLIENT" "$M2_CANARY_SERVER" "$M2_CANARY_SPARK" "$M2_CANARY_HADOOP")
@@ -161,9 +173,25 @@ fi
 # coordinate. Applied as a persistent setting so it sticks across the two sbt invocations below.
 SET_VERSION_CMD="set ThisBuild / version := \"$UC_VERSION\""
 
+# Force publishLocal / publishM2 to overwrite existing artifacts. UC artifacts at the same
+# coordinate can be left behind from a prior run (e.g. cross-Spark publish re-invokes this
+# script for a different sparkVersion while client/server/hadoop are already in ~/.ivy2/local
+# and ~/.m2 from the first invocation). publishLocalConfiguration / publishM2Configuration are
+# task settings scoped per-project (ThisBuild / Global don't propagate), so we set them on each
+# project we publish. Both configs need overriding: publishLocal uses the former, publishM2
+# uses the latter.
+SET_OVERWRITE_CMDS=()
+for p in client server hadoop spark; do
+  SET_OVERWRITE_CMDS+=(
+    "set $p / publishLocalConfiguration := ($p / publishLocalConfiguration).value.withOverwrite(true)"
+    "set $p / publishM2Configuration := ($p / publishM2Configuration).value.withOverwrite(true)"
+  )
+done
+
 echo ">>> Building and publishing UC client + server to local Maven repo"
 ./build/sbt \
   "$SET_VERSION_CMD" \
+  "${SET_OVERWRITE_CMDS[@]}" \
   "set client / Compile / packageDoc / publishArtifact := false" \
   clean \
   client/generate \
@@ -175,10 +203,18 @@ echo ">>> Building and publishing UC client + server to local Maven repo"
   hadoop/publishM2
 
 # spark/publishM2 can hit a transient coursier lock race - retry up to 3 times.
-echo ">>> Building and publishing UC spark module to local Maven repo"
+# -DsparkVersion picks the Spark variant UC builds against; the resulting artifact is
+# unitycatalog-spark_${SPARK_VERSION}_2.13, matching UC_SPARK_ARTIFACT above.
+# -DskipDeltaSpark=true drops UC's test-only delta-spark_X.Y dep, which would otherwise
+# fail resolution: we publish UC before Delta itself is built, and on snapshot Spark
+# versions (e.g. 4.2) no matching delta-spark release exists at all.
+echo ">>> Building and publishing UC spark module (Spark $SPARK_VERSION) to local Maven repo"
 for attempt in 1 2 3; do
   if ./build/sbt \
+    -DsparkVersion="$SPARK_VERSION" \
+    -DskipDeltaSpark=true \
     "$SET_VERSION_CMD" \
+    "${SET_OVERWRITE_CMDS[@]}" \
     "set client / Compile / packageDoc / publishArtifact := false" \
     spark/publishLocal \
     spark/publishM2; then