kernel_test.yaml

name: "Delta Kernel"

on:
  push:
    branches: [master, branch-*]
    paths-ignore:
      - '**.md'
      - '**.txt'
  pull_request:
    branches: [master, branch-*]
    paths-ignore:
      - '**.md'
      - '**.txt'

# Cancel previous runs when new commits are pushed
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # Point SBT to our cache directories for consistency
  SBT_OPTS: "-Dsbt.coursier.home-dir=/home/runner/.cache/coursier -Dsbt.ivy.home=/home/runner/.ivy2"

jobs:
  test:
    name: "DK: Shard ${{ matrix.shard }}"
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false # Allow all shards to run even if one fails
      matrix:
        shard: [0, 1, 2, 3]
    env:
      SCALA_VERSION: 2.13.16
      NUM_SHARDS: 4
      DISABLE_UNIDOC: true # Another unidoc workflow will test unidoc.
      TEST_PARALLELISM_COUNT: 4
    steps:
      - name: Show runner specs
        run: |
          echo "=== GitHub Runner Specs ==="
          echo "CPU cores: $(nproc)"
          echo "CPU info: $(lscpu | grep 'Model name' | cut -d':' -f2 | xargs)"
          echo "Total RAM: $(free -h | grep '^Mem:' | awk '{print $2}')"
          echo "Available RAM: $(free -h | grep '^Mem:' | awk '{print $7}')"
          echo "Disk space: $(df -h / | tail -1 | awk '{print $2 " total, " $4 " available"}')"
          echo "Runner OS: ${{ runner.os }}"
          echo "Runner arch: ${{ runner.arch }}"
      - name: Checkout code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17.
      - name: install java
        uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0
        with:
          distribution: "zulu"
          java-version: "17"
      - name: Restore SBT cache
        id: cache-sbt
        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          path: |
            ~/.sbt
            ~/.ivy2
            ~/.cache/coursier
          key: sbt-kernel-${{ runner.os }}-scala${{ env.SCALA_VERSION }}
      - name: Check cache status
        run: |
          if [ "${{ steps.cache-sbt.outputs.cache-hit }}" == "true" ]; then
            echo "✅ Cache HIT - using cached dependencies"
          else
            echo "❌ Cache MISS - will download dependencies"
          fi
      # run-tests.py invokes sbt with `++ 2.13.16`, which triggers cross-version dependency resolution
      # across every project (including kernelUnityCatalog). Publish the pinned UC build locally first
      # so that resolution doesn't miss.
      - name: Set up pinned Unity Catalog
        uses: ./.github/actions/setup-unitycatalog
      - name: Run unit tests
        run: |
          python run-tests.py --group kernel --coverage --shard ${{ matrix.shard }}
      - name: Save SBT cache
        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          path: |
            ~/.sbt
            ~/.ivy2
            ~/.cache/coursier
          key: sbt-kernel-${{ runner.os }}-scala${{ env.SCALA_VERSION }}

  integration-test:
    name: "DK: Integration"
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
      # The integration test itself runs on JDK 11 (no Spark dependency), but UC's sbt build needs
      # JDK 17, so we install 17 first, publish UC, then switch the active JDK to 11 for the actual
      # test run.
      - name: install java 17 for UC build
        uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1
        with:
          distribution: "zulu"
          java-version: "17"
      - name: Set up pinned Unity Catalog
        uses: ./.github/actions/setup-unitycatalog
      - name: install java 11 for integration test
        uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3.14.1
        with:
          distribution: "zulu"
          java-version: "11"
      - name: Run integration tests
        run: |
          cd kernel/examples && python run-kernel-examples.py --use-local