Initial cherry-pick of changes

Author: allisonport-db

.github/workflows/connectors_test.yaml

@@ -1,3 +1,4 @@
+# TODO - do we need to update this?
 name: "Delta Connectors"
 on: [push, pull_request]
 jobs:

.github/workflows/kernel_test.yaml

@@ -5,17 +5,30 @@ jobs:
     name: "DK"
     runs-on: ubuntu-24.04
     env:
-      SCALA_VERSION: 2.12.18
+      SCALA_VERSION: 2.13.13
     steps:
       - uses: actions/checkout@v3
+
+      # Install JDK 8
       - name: install java
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
           java-version: "8"
-      - name: Run tests
-        run: |
-          python run-tests.py --group kernel --coverage
-      - name: Run integration tests
+
+      # Run integration tests with JDK 8, as they have no Spark dependency
+      - name: Run integration tests (JDK 8)
         run: |
           cd kernel/examples && python run-kernel-examples.py --use-local
+
+      # Install JDK 17
+      - name: install java
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: "17"
+
+      # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17.
+      - name: Run unit tests (JDK 17)
+        run: |
+          python run-tests.py --group kernel --coverage

.github/workflows/spark_examples_test.yaml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       matrix:
         # These Scala versions must match those in the build.sbt
-        scala: [2.12.18, 2.13.13]
+        scala: [2.13.13]
     env:
       SCALA_VERSION: ${{ matrix.scala }}
     steps:
@@ -24,7 +24,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "8"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:
@@ -44,13 +44,10 @@ jobs:
           sudo apt install libedit-dev
         if: steps.git-diff.outputs.diff
       - name: Run Delta Spark Local Publishing and Examples Compilation
-        # examples/scala/build.sbt will compile against the local Delta relase version (e.g. 3.2.0-SNAPSHOT).
+        # examples/scala/build.sbt will compile against the local Delta release version (e.g. 3.2.0-SNAPSHOT).
         # Thus, we need to publishM2 first so those jars are locally accessible.
-        # We publish storage explicitly so that it is available for the Scala 2.13 build. As a java project
-        # it is typically only released when publishing for Scala 2.12.
         run: |
           build/sbt clean
-          build/sbt storage/publishM2
           build/sbt "++ $SCALA_VERSION publishM2"
           cd examples/scala && build/sbt "++ $SCALA_VERSION compile"
         if: steps.git-diff.outputs.diff

.github/workflows/spark_python_test.yaml renamed to .github/workflows/spark_master_python_test.yaml

@@ -1,4 +1,4 @@
-name: "Delta Spark Python"
+name: "Delta Spark Master Python"
 on: [push, pull_request]
 jobs:
   test:
@@ -7,7 +7,7 @@ jobs:
     strategy:
       matrix:
         # These Scala versions must match those in the build.sbt
-        scala: [2.12.18]
+        scala: [2.13.13]
     env:
       SCALA_VERSION: ${{ matrix.scala }}
     steps:
@@ -24,19 +24,20 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "8"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:
           path: |
             ~/.sbt
             ~/.ivy2
             ~/.cache/coursier
+            !~/.cache/coursier/v1/https/repository.apache.org/content/groups/snapshots
           # Change the key if dependencies are changed. For each key, GitHub Actions will cache the
           # the above directories when we use the key for the first time. After that, each run will
           # just use the cache. The cache is immutable so we need to use a new key when trying to
           # cache new stuff.
-          key: delta-sbt-cache-spark3.2-scala${{ matrix.scala }}
+          key: delta-sbt-cache-spark-master-scala${{ matrix.scala }}
       - name: Install Job dependencies
         run: |
           sudo apt-get update
@@ -47,34 +48,27 @@ jobs:
           tar -xvzf buf-Linux-x86_64.tar.gz -C ~/buf --strip-components 1
           rm buf-Linux-x86_64.tar.gz
           sudo apt install python3-pip --fix-missing
-          sudo pip3 install pipenv==2024.4.1
+          sudo pip3 install pipenv==2021.5.29
           curl https://pyenv.run | bash
           export PATH="~/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
           eval "$(pyenv virtualenv-init -)"
-          pyenv install 3.8.18
-          pyenv global system 3.8.18
-          pipenv --python 3.8 install
-          # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version
-          # available. From version 24.1, `pip` doesn't allow installing python packages
-          # with version string containing `-`. In Delta-Spark case, the pypi package generated has
-          # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from
-          # the`version.sbt` file.
-          pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0
-          pipenv run pip install pyspark==3.5.3
-          pipenv run pip install flake8==3.5.0 pypandoc==1.3.3
+          pyenv install 3.9
+          pyenv global system 3.9
+          pipenv --python 3.9 install
+          pipenv run pip install flake8==3.9.0
           pipenv run pip install black==23.9.1
-          pipenv run pip install importlib_metadata==3.10.0
-          pipenv run pip install mypy==0.982
+          pipenv run pip install mypy==1.8.0
           pipenv run pip install mypy-protobuf==3.3.0
           pipenv run pip install cryptography==37.0.4
           pipenv run pip install twine==4.0.1
           pipenv run pip install wheel==0.33.4
           pipenv run pip install setuptools==41.1.0
           pipenv run pip install pydocstyle==3.0.0
-          pipenv run pip install pandas==1.1.3
+          pipenv run pip install pandas==1.4.4
           pipenv run pip install pyarrow==8.0.0
-          pipenv run pip install numpy==1.20.3
+          pipenv run pip install numpy==1.21
+          pipenv run pip install https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc4-bin//pyspark-4.0.0.dev1.tar.gz
         if: steps.git-diff.outputs.diff
       - name: Run Python tests
         # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_master_test.yaml

.github/workflows/spark_master_test.yaml

@@ -43,15 +43,43 @@ jobs:
           # cache new stuff.
           key: delta-sbt-cache-spark-master-scala${{ matrix.scala }}
       - name: Install Job dependencies
+        # TODO: update pyspark installation once Spark preview is formally released
         run: |
           sudo apt-get update
           sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git
           sudo apt install libedit-dev
+          curl -LO https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz
+          mkdir -p ~/buf
+          tar -xvzf buf-Linux-x86_64.tar.gz -C ~/buf --strip-components 1
+          rm buf-Linux-x86_64.tar.gz
+          sudo apt install python3-pip --fix-missing
+          sudo pip3 install pipenv==2021.5.29
+          curl https://pyenv.run | bash
+          export PATH="~/.pyenv/bin:$PATH"
+          eval "$(pyenv init -)"
+          eval "$(pyenv virtualenv-init -)"
+          pyenv install 3.9
+          pyenv global system 3.9
+          pipenv --python 3.9 install
+          pipenv run pip install flake8==3.9.0
+          pipenv run pip install black==23.9.1
+          pipenv run pip install mypy==1.8.0
+          pipenv run pip install mypy-protobuf==3.3.0
+          pipenv run pip install cryptography==37.0.4
+          pipenv run pip install twine==4.0.1
+          pipenv run pip install wheel==0.33.4
+          pipenv run pip install setuptools==41.1.0
+          pipenv run pip install pydocstyle==3.0.0
+          pipenv run pip install pandas==1.4.4
+          pipenv run pip install pyarrow==8.0.0
+          pipenv run pip install numpy==1.21
+          pipenv run pip install https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc4-bin//pyspark-4.0.0.dev1.tar.gz
         if: steps.git-diff.outputs.diff
       - name: Run Spark Master tests
         # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_test.yaml
+        # NOTE: in this branch, the default sparkVersion is the SPARK_MASTER_VERSION
         run: |
-          TEST_PARALLELISM_COUNT=4 SHARD_ID=${{matrix.shard}} build/sbt -DsparkVersion=master "++ ${{ matrix.scala }}" clean spark/test
           TEST_PARALLELISM_COUNT=4 build/sbt -DsparkVersion=master "++ ${{ matrix.scala }}" clean connectServer/test
           TEST_PARALLELISM_COUNT=4 build/sbt -DsparkVersion=master "++ ${{ matrix.scala }}" clean connectServer/assembly connectClient/test
+          TEST_PARALLELISM_COUNT=4 pipenv run python run-tests.py --group spark
         if: steps.git-diff.outputs.diff

.github/workflows/spark_test.yaml

@@ -1,3 +1,4 @@
+# TODO - address unidoc build
   name: "Unidoc"
   on: [push, pull_request]
   jobs:
@@ -6,14 +7,14 @@
       runs-on: ubuntu-24.04
       strategy:
         matrix:
-          # These Scala versions must match those in the build.sbt
-          scala: [2.13.13, 2.12.18]
+        # These Scala versions must match those in the build.sbt
+          scala: [2.13.13]
       steps:
         - name: install java
           uses: actions/setup-java@v3
           with:
             distribution: "zulu"
-            java-version: "8"
+            java-version: "17"
         - uses: actions/checkout@v3
         - name: generate unidoc
           run: build/sbt "++ ${{ matrix.scala }}" unidoc