Fix UC setup cache key collisions and local Spark-version switching #9629
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Delta Build" | |
| on: | |
| push: | |
| branches: [master, branch-*] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| pull_request: | |
| branches: [master, branch-*] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| jobs: | |
| test: | |
| name: "Build Test" | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| - name: Install Java 17 | |
| uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 | |
| with: | |
| distribution: "zulu" | |
| java-version: "17" | |
| - name: Set up Python | |
| uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1 | |
| with: | |
| python-version: '3.9' | |
| - name: Restore SBT cache | |
| uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: delta-sbt-cache-cross-spark-${{ hashFiles('project/scripts/setup_unitycatalog_main.sh') }} | |
| # publishM2 compiles every aggregated project, including storage, which has | |
| # unitycatalog-client as a compile-scope dependency. test_cross_spark_publish.py also | |
| # iterates over released Spark versions (sbt -DsparkVersion=<X.Y>), so we need UC's | |
| # spark connector published for each variant Delta will resolve. Invoke the setup action | |
| # once per Spark variant; the action's cache key partitions by SPARK_VERSION so each | |
| # variant warms its own cache entry. Keep this list in sync with the released versions | |
| # in project/spark-versions.json (snapshot versions are skipped by the cross-Spark test). | |
| - name: Set up pinned Unity Catalog (Spark 4.0) | |
| uses: ./.github/actions/setup-unitycatalog | |
| env: | |
| SPARK_VERSION: "4.0" | |
| - name: Set up pinned Unity Catalog (Spark 4.1) | |
| uses: ./.github/actions/setup-unitycatalog | |
| env: | |
| SPARK_VERSION: "4.1" | |
| - name: Run cross-Spark build test | |
| run: python project/tests/test_cross_spark_publish.py | |
| - name: Save SBT cache | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/master' | |
| uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: delta-sbt-cache-cross-spark-${{ hashFiles('project/scripts/setup_unitycatalog_main.sh') }} |