Fix UC setup cache key collisions and local Spark-version switching #24200
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Delta Kernel" | |
| on: | |
| push: | |
| branches: [master, branch-*] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| pull_request: | |
| branches: [master, branch-*] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| # Cancel previous runs when new commits are pushed | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # Point SBT to our cache directories for consistency | |
| SBT_OPTS: "-Dsbt.coursier.home-dir=/home/runner/.cache/coursier -Dsbt.ivy.home=/home/runner/.ivy2" | |
| jobs: | |
| test: | |
| name: "DK: Shard ${{ matrix.shard }}" | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false # Allow all shards to run even if one fails | |
| matrix: | |
| shard: [0, 1, 2, 3] | |
| env: | |
| SCALA_VERSION: 2.13.16 | |
| NUM_SHARDS: 4 | |
| DISABLE_UNIDOC: true # Another unidoc workflow will test unidoc. | |
| TEST_PARALLELISM_COUNT: 4 | |
| steps: | |
| - name: Show runner specs | |
| run: | | |
| echo "=== GitHub Runner Specs ===" | |
| echo "CPU cores: $(nproc)" | |
| echo "CPU info: $(lscpu | grep 'Model name' | cut -d':' -f2 | xargs)" | |
| echo "Total RAM: $(free -h | grep '^Mem:' | awk '{print $2}')" | |
| echo "Available RAM: $(free -h | grep '^Mem:' | awk '{print $7}')" | |
| echo "Disk space: $(df -h / | tail -1 | awk '{print $2 " total, " $4 " available"}')" | |
| echo "Runner OS: ${{ runner.os }}" | |
| echo "Runner arch: ${{ runner.arch }}" | |
| - name: Checkout code | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 | |
| # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17. | |
| - name: install java | |
| uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 | |
| with: | |
| distribution: "zulu" | |
| java-version: "17" | |
| - name: Restore SBT cache | |
| id: cache-sbt | |
| uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: sbt-kernel-${{ runner.os }}-scala${{ env.SCALA_VERSION }} | |
| - name: Check cache status | |
| run: | | |
| if [ "${{ steps.cache-sbt.outputs.cache-hit }}" == "true" ]; then | |
| echo "✅ Cache HIT - using cached dependencies" | |
| else | |
| echo "❌ Cache MISS - will download dependencies" | |
| fi | |
| # run-tests.py invokes sbt with `++ 2.13.16`, which triggers cross-version dependency resolution | |
| # across every project (including kernelUnityCatalog). Publish the pinned UC build locally first | |
| # so that resolution doesn't miss. | |
| - name: Set up pinned Unity Catalog | |
| uses: ./.github/actions/setup-unitycatalog | |
| - name: Run unit tests | |
| run: | | |
| python run-tests.py --group kernel --coverage --shard ${{ matrix.shard }} | |
| - name: Save SBT cache | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/master' | |
| uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: | | |
| ~/.sbt | |
| ~/.ivy2 | |
| ~/.cache/coursier | |
| key: sbt-kernel-${{ runner.os }}-scala${{ env.SCALA_VERSION }} | |
| integration-test: | |
| name: "DK: Integration" | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 | |
| # The integration test itself runs on JDK 11 (no Spark dependency), but UC's sbt build needs | |
| # JDK 17, so we install 17 first, publish UC, then switch the active JDK to 11 for the actual | |
| # test run. | |
| - name: install java 17 for UC build | |
| uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 | |
| with: | |
| distribution: "zulu" | |
| java-version: "17" | |
| - name: Set up pinned Unity Catalog | |
| uses: ./.github/actions/setup-unitycatalog | |
| - name: install java 11 for integration test | |
| uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1 | |
| with: | |
| distribution: "zulu" | |
| java-version: "11" | |
| - name: Run integration tests | |
| run: | | |
| cd kernel/examples && python run-kernel-examples.py --use-local |