Support DBR16.3 with GDAL 3.10

.github/actions/scala_build/action.yml

@@ -13,32 +13,34 @@ runs:
             java-version: '8'
             distribution: 'zulu'
         - name: Configure python interpreter
-          uses: actions/setup-python@v2
-          with:
-            python-version: ${{ matrix.python }}
+          shell: bash
+          run: |
+            sudo bash <<EOF
+              add-apt-repository ppa:deadsnakes/ppa
+              apt update
+              apt install -y python${{ matrix.python_minor }} \
+              python${{ matrix.python_minor }}-dev \
+              python${{ matrix.python_minor }}-venv \
+              python${{ matrix.python_minor }}-distutils
+              mkdir -p /databricks
+              chown $USER:$USER /databricks
+              chmod -R 766 /databricks/
+            EOF
+            python${{ matrix.python_minor }} -m venv /databricks/python3
+            /databricks/python3/bin/pip install numpy==${{ matrix.numpy }}
         - name: Add packaged GDAL dependencies
           shell: bash
           run : |
-            # - update apt
-            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc)-backports main universe multiverse restricted"
-            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc)-updates main universe multiverse restricted"
-            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc)-security main multiverse restricted universe"
-            sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc) main multiverse restricted universe"
-            sudo apt-get update -y
-            # - install natives
-            sudo apt-get install -y unixodbc libcurl3-gnutls libsnappy-dev libopenjp2-7
-            sudo apt-get install -y gdal-bin libgdal-dev python3-numpy python3-gdal zip unzip
-            # - install pip libs
-            pip install --upgrade pip
-            pip install gdal==${{ matrix.gdal }}
-            # - add the so files 
-            sudo wget -nv -P /usr/lib -nc https://raw.githubusercontent.com/databrickslabs/mosaic/main/resources/gdal/jammy/libgdalalljni.so
-            sudo wget -nv -P /usr/lib -nc https://raw.githubusercontent.com/databrickslabs/mosaic/main/resources/gdal/jammy/libgdalalljni.so.30
-            sudo wget -nv -P /usr/lib -nc https://raw.githubusercontent.com/databrickslabs/mosaic/main/resources/gdal/jammy/libgdalalljni.so.30.0.3
+            cd python && /databricks/python3/bin/pip install .
+    #            echo "/databricks/python3/bin" >> "$GITHUB_PATH"
         - name: Test and build the scala JAR - skip tests is false
           if: inputs.skip_tests == 'false'
           shell: bash
-          run: sudo mvn -q clean install
+          run: |
+            source /databricks/python3/bin/activate
+            readlink -f $(which python3)
+            python3 -c 'import numpy as np; print(np.__version__)'
+            sudo mvn -q clean install
         - name: Build the scala JAR - skip tests is true
           if: inputs.skip_tests == 'true'
           shell: bash

.github/workflows/build_main.yml

@@ -1,4 +1,4 @@
-name: build main
+name: build main 
 on:
   push:
     branches-ignore:
@@ -16,11 +16,12 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.10.12 ]
-        numpy: [ 1.22.4 ]
-        gdal: [ 3.4.1 ]
-        spark: [ 3.4.1 ]
-        R: [ 4.2.2 ]
+        python: [ 3.12.3 ]
+        python_minor: [ 3.12 ]
+        numpy: [ 1.26.4 ]
+        gdal: [ 3.10.1 ]
+        spark: [ 3.5.2 ]
+        R: [ 4.4.0 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2

.github/workflows/build_python.yml

@@ -12,11 +12,11 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.10.12 ]
-        numpy: [ 1.22.4 ]
-        gdal: [ 3.4.1 ]
-        spark: [ 3.4.1 ]
-        R: [ 4.2.2 ]
+        python: [ 3.12.3 ]
+        numpy: [ 1.26.4 ]
+        gdal: [ 3.10.1 ]
+        spark: [ 3.5.2 ]
+        R: [ 4.4.0 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2

.github/workflows/build_r.yml

@@ -13,11 +13,11 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.10.12 ]
-        numpy: [ 1.22.4 ]
-        gdal: [ 3.4.1 ]
-        spark: [ 3.4.1 ]
-        R: [ 4.2.2 ]
+        python: [ 3.12.3 ]
+        numpy: [ 1.26.4 ]
+        gdal: [ 3.10.1 ]
+        spark: [ 3.5.2 ]
+        R: [ 4.4.0 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2

.github/workflows/build_scala.yml

@@ -11,11 +11,11 @@ jobs:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       matrix:
-        python: [ 3.10.12 ]
-        numpy: [ 1.22.4 ]
-        gdal: [ 3.4.1 ]
-        spark: [ 3.4.1 ]
-        R: [ 4.2.2 ]
+        python: [ 3.12.3 ]
+        numpy: [ 1.26.4 ]
+        gdal: [ 3.10.1 ]
+        spark: [ 3.5.2 ]
+        R: [ 4.4.0 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2

.github/workflows/release.yml

@@ -15,11 +15,11 @@ jobs:
       contents: write
     strategy:
       matrix:
-        python: [ 3.10.12 ]
-        numpy: [ 1.22.4 ]
-        gdal: [ 3.4.1 ]
-        spark: [ 3.4.1 ]
-        R: [ 4.2.2 ]
+        python: [ 3.12.3 ]
+        numpy: [ 1.26.4 ]
+        gdal: [ 3.10.1 ]
+        spark: [ 3.5.2 ]
+        R: [ 4.4.0 ]
     steps:
       - name: checkout code
         uses: actions/checkout@v2

R/sparkR-mosaic/sparkrMosaic/DESCRIPTION

@@ -1,6 +1,6 @@
 Package: sparkrMosaic
 Title: SparkR bindings for Databricks Mosaic
-Version: 0.4.3
+Version: 0.4.4
 Authors@R:
     person("Robert", "Whiffin", , "[email protected]", role = c("aut", "cre")
            )

R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION

@@ -1,6 +1,6 @@
 Package: sparklyrMosaic
 Title: sparklyr bindings for Databricks Mosaic
-Version: 0.4.3
+Version: 0.4.4
 Authors@R:
     person("Robert", "Whiffin", , "[email protected]", role = c("aut", "cre")
            )

R/sparklyr-mosaic/tests.R

@@ -9,7 +9,7 @@ library(sparklyr.nested)
 spark_home <- Sys.getenv("SPARK_HOME")
 spark_home_set(spark_home)
 
-install.packages("sparklyrMosaic_0.4.3.tar.gz", repos = NULL)
+install.packages("sparklyrMosaic_0.4.4.tar.gz", repos = NULL)
 library(sparklyrMosaic)
 
 # find the mosaic jar in staging

docs/source/conf.py

@@ -22,7 +22,7 @@
 author = 'Milos Colic, Stuart Lynn, Michael Johns, Robert Whiffin'
 
 # The full version, including alpha/beta/rc tags
-release = "v0.4.3"
+release = "v0.4.4"
 
 
 # -- General configuration ---------------------------------------------------

pom.xml

@@ -83,17 +83,42 @@
         </dependency>
         <dependency>
             <groupId>io.delta</groupId>
-            <artifactId>delta-core_${scala.compat.version}</artifactId>
-            <version>2.4.0</version>
+            <artifactId>delta-spark_${scala.compat.version}</artifactId>
+            <version>3.2.0</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.15.2</version>
+            <scope>provided</scope>
+        </dependency>
+        <!-- log4j -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>${slf4j.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>jul-to-slf4j</artifactId>
+            <version>${slf4j.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>jcl-over-slf4j</artifactId>
+            <version>${slf4j.version}</version>
+            <scope>provided</scope>
+            <!-- runtime scope is appropriate, but causes SBT build problems -->
+        </dependency>
         <!--geo dependencies-->
         <dependency>
             <groupId>com.uber</groupId>
             <artifactId>h3</artifactId>
-            <!--H3 fixed at 3.7.0 until Photon updates the version-->
             <!--suppress MavenPackageUpdate-->
-            <version>3.7.0</version>
+            <version>3.7.3</version>
         </dependency>
         <dependency>
             <groupId>org.locationtech.jts</groupId>
@@ -118,7 +143,7 @@
         <dependency>
             <groupId>org.gdal</groupId>
             <artifactId>gdal</artifactId>
-            <version>3.4.0</version>
+            <version>3.10.1</version>
         </dependency>
 
     </dependencies>
@@ -174,14 +199,32 @@
                 <configuration>
                     <!-- Tests will be run with scalatest-maven-plugin instead -->
                     <skipTests>true</skipTests>
-                    <argLine>@{argLine} -Djava.library.path=/usr/local/lib;/usr/java/packages/lib;/usr/lib64;/lib64;/lib;/usr/lib</argLine>
+                    <argLine>@{argLine} -Djava.library.path=/usr/local/lib;/usr/java/packages/lib;/usr/lib64;/lib64;/lib;/usr/lib;/usr/lib/x86_64-linux-gnu;/usr/lib/x86_64-linux-gnu/jni</argLine>
                 </configuration>
             </plugin>
             <plugin>
                 <groupId>org.scalatest</groupId>
                 <artifactId>scalatest-maven-plugin</artifactId>
                 <version>2.0.0</version>
                 <configuration>
+                    <environmentVariables>
+                        <JAVA_TOOL_OPTIONS>
+                            --add-opens=java.base/java.lang=ALL-UNNAMED
+                            --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+                            --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+                            --add-opens=java.base/java.io=ALL-UNNAMED
+                            --add-opens=java.base/java.net=ALL-UNNAMED
+                            --add-opens=java.base/java.nio=ALL-UNNAMED
+                            --add-opens=java.base/java.util=ALL-UNNAMED
+                            --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+                            --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+                            --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+                            --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+                            --add-opens=java.base/sun.security.action=ALL-UNNAMED
+                            --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+                            --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
+                        </JAVA_TOOL_OPTIONS>
+                    </environmentVariables>
                     <reportsDirectory>${project.build.directory}/test-reports</reportsDirectory>
                 </configuration>
                 <executions>
@@ -256,8 +299,9 @@
             <properties>
                 <scala.version>2.12.10</scala.version>
                 <scala.compat.version>2.12</scala.compat.version>
-                <spark.version>3.4.1</spark.version>
-                <mosaic.version>0.4.3</mosaic.version>
+                <spark.version>3.5.0</spark.version>
+                <slf4j.version>2.0.7</slf4j.version>
+                <mosaic.version>0.4.4</mosaic.version>
             </properties>
             <build>
                 <plugins>
@@ -289,10 +333,11 @@
             <!-- local testing `mvn test -PskipScoverage -DskipTests=false -Dsuite=...` -->
             <id>skipScoverage</id>
             <properties>
-                <scala.version>2.12.10</scala.version>
+                <scala.version>2.12.18</scala.version>
                 <scala.compat.version>2.12</scala.compat.version>
-                <spark.version>3.4.1</spark.version>
-                <mosaic.version>0.4.3</mosaic.version>
+                <spark.version>3.5.2</spark.version>
+                <slf4j.version>2.0.7</slf4j.version>
+                <mosaic.version>0.4.4</mosaic.version>
                 <maven.compiler.source>1.8</maven.compiler.source>
                 <maven.compiler.target>1.8</maven.compiler.target>
             </properties>

python/mosaic/__init__.py

@@ -4,4 +4,4 @@
 from .readers import read
 from .utils.display_handler import displayMosaic
 
-__version__ = "0.4.3"
+__version__ = "0.4.4"

python/mosaic/api/accessors.py

@@ -1,5 +1,5 @@
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 
 from mosaic.config import config
 from mosaic.utils.types import ColumnOrName

python/mosaic/api/aggregators.py

@@ -1,5 +1,5 @@
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 
 from mosaic.config import config
 from mosaic.utils.types import ColumnOrName

python/mosaic/api/constructors.py

@@ -1,5 +1,5 @@
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 
 from mosaic.config import config
 from mosaic.utils.types import ColumnOrName

python/mosaic/api/functions.py

@@ -1,7 +1,8 @@
 from typing import Any
 
 from pyspark.sql import Column
-from pyspark.sql.functions import _to_java_column as pyspark_to_java_column
+
+from pyspark.sql.column import _to_java_column as pyspark_to_java_column
 from pyspark.sql.functions import lit
 
 from mosaic.config import config