Fix UC setup cache key collisions and local Spark-version switching #23751
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Delta Spark" | |
| on: | |
| push: | |
| branches: [master, branch-*] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| pull_request: | |
| branches: [master, branch-*] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| jobs: | |
| # Generate Spark versions matrix from CrossSparkVersions.scala | |
| # This ensures the workflow always uses the versions defined in the build | |
| generate-matrix: | |
| name: "Generate Spark Versions Matrix" | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| spark_versions: ${{ steps.generate.outputs.spark_versions }} | |
| steps: | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| - name: install java | |
| uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 | |
| with: | |
| distribution: "zulu" | |
| java-version: "17" | |
| - name: Generate Spark versions matrix | |
| id: generate | |
| run: | | |
| # The script automatically generates spark-versions.json from CrossSparkVersions.scala | |
| SPARK_VERSIONS=$(python3 project/scripts/get_spark_version_info.py --all-spark-versions) | |
| echo "spark_versions=$SPARK_VERSIONS" >> $GITHUB_OUTPUT | |
| echo "Generated Spark versions: $SPARK_VERSIONS" | |
| test: | |
| name: "DS: Spark ${{ matrix.spark_version }}, Scala ${{ matrix.scala }}, Shard ${{ matrix.shard }}" | |
| runs-on: ubuntu-24.04 | |
| needs: generate-matrix | |
| # Spark 4.2 is a preview release until the Spark 4.2 release is published. | |
| # Do not block PRs on its failures while the preview lane is stabilizing. | |
| continue-on-error: ${{ matrix.spark_version == '4.2' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Spark versions are dynamically generated from CrossSparkVersions.scala | |
| # DO NOT hardcode versions here - they are automatically loaded from the build configuration | |
| spark_version: ${{ fromJson(needs.generate-matrix.outputs.spark_versions) }} | |
| # These Scala versions must match those in the build.sbt | |
| scala: [2.13.16] | |
| # Important: This list of shards must be [0..NUM_SHARDS - 1] | |
| shard: [0, 1, 2, 3, 4, 5, 6, 7] | |
| env: | |
| SCALA_VERSION: ${{ matrix.scala }} | |
| SPARK_VERSION: ${{ matrix.spark_version }} | |
| # Important: This must be the same as the length of shards in matrix | |
| NUM_SHARDS: 8 | |
| steps: | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| - name: Get Spark version details | |
| id: spark-details | |
| run: | | |
| # The script automatically generates spark-versions.json if needed | |
| JVM_VERSION=$(python3 project/scripts/get_spark_version_info.py --get-field "${{ matrix.spark_version }}" targetJvm | jq -r) | |
| echo "jvm_version=$JVM_VERSION" >> $GITHUB_OUTPUT | |
| echo "Using JVM version: $JVM_VERSION for Spark ${{ matrix.spark_version }}" | |
| - name: install java | |
| uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 | |
| with: | |
| distribution: "zulu" | |
| java-version: ${{ steps.spark-details.outputs.jvm_version }} | |
| - name: Restore SBT cache | |
| uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: delta-sbt-cache-spark${{ matrix.spark_version }}-scala${{ matrix.scala }} | |
| # Delta's sparkUnityCatalog module (part of sparkGroup) depends on APIs that are only in | |
| # unreleased UC. Publish the pinned UC build locally before sbt tries to resolve it. | |
| - name: Set up pinned Unity Catalog | |
| uses: ./.github/actions/setup-unitycatalog | |
| - name: Scala structured logging style check | |
| run: | | |
| if [ -f ./dev/spark_structured_logging_style.py ]; then | |
| python3 ./dev/spark_structured_logging_style.py | |
| fi | |
| - name: Run Scala/Java tests | |
| run: | | |
| TEST_PARALLELISM_COUNT=4 python3 run-tests.py --group spark --shard ${{ matrix.shard }} --spark-version ${{ matrix.spark_version }} | |
| - name: Upload test reports | |
| if: always() | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: test-reports-spark${{ matrix.spark_version }}-shard${{ matrix.shard }} | |
| path: "**/target/test-reports/*.xml" | |
| retention-days: 7 | |
| - name: Save SBT cache | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/master' | |
| uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: delta-sbt-cache-spark${{ matrix.spark_version }}-scala${{ matrix.scala }} |