chore(deps): update delta-spark to 3.3.0 for local and remote pyspark (#11123)

## Description of changes

1. Upgrade `delta-spark` from v3.2.1 to
[v3.3.0](https://github.com/delta-io/delta/releases/tag/v3.3.0) for
pyspark backend testing in github actions
2. Upgrade `delta-spark` [maven
package](https://mvnrepository.com/artifact/io.delta/delta-spark) from
v.2.1.0 to v3.3.0 in spark-connect container 🐳
- Ensure consistency w/ local pyspark testing &
[compatibility](https://docs.delta.io/latest/releases.html#compatibility-with-apache-spark)
with pyspark v3.5.5
3. Upgrade spark-connect configuration to enable proper delta & catalog
functionality

----
- **2** + **3** together resolved spark-connect testing issues from
#11120

---------

Co-authored-by: Phillip Cloud <[email protected]>

Author: jakepenzak

.github/workflows/ibis-backends.yml

@@ -561,20 +561,20 @@ jobs:
             pyspark-minor-version: "3.5"
             tag: local
             deps:
-              - delta-spark==3.2.1
+              - delta-spark==3.3.0
           - python-version: "3.13"
             pyspark-minor-version: "3.5"
             tag: local
             deps:
               - setuptools==75.1.0
-              - delta-spark==3.2.1
+              - delta-spark==3.3.0
           - python-version: "3.12"
             pyspark-minor-version: "3.5"
             SPARK_REMOTE: "sc://localhost:15002"
             tag: remote
             deps:
               - setuptools==75.1.0
-              - delta-spark==3.2.1
+              - delta-spark==3.3.0
               - googleapis-common-protos
               - grpcio
               - grpcio-status

compose.yaml

@@ -593,7 +593,7 @@ services:
     image: bitnami/spark:3.5.5
     ports:
       - 15002:15002
-    command: /opt/bitnami/spark/sbin/start-connect-server.sh --name ibis_testing --packages org.apache.spark:spark-connect_2.12:3.5.3,org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,io.delta:delta-core_2.12:2.1.0
+    command: /opt/bitnami/spark/sbin/start-connect-server.sh --name ibis_testing --packages org.apache.spark:spark-connect_2.12:3.5.3,org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,io.delta:delta-spark_2.12:3.3.0
     healthcheck:
       test:
         - CMD-SHELL

docker/spark-connect/conf.properties

@@ -10,3 +10,5 @@ spark.sql.session.timeZone=UTC
 spark.sql.streaming.schemaInference=true
 spark.ui.enabled=false
 spark.ui.showConsoleProgress=false
+spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension
+spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog

ibis/backends/pyspark/tests/conftest.py

@@ -288,6 +288,8 @@ def connect(*, tmpdir, worker_id, **kw):
                 )
             ).open(mode="r") as config_file:
                 for line in config_file:
+                    if "delta" in line:
+                        continue
                     config = config.config(*map(str.strip, line.strip().split("=", 1)))
 
             config = (

ibis/backends/tests/test_export.py

@@ -27,7 +27,7 @@
     SnowflakeProgrammingError,
     TrinoUserError,
 )
-from ibis.conftest import CI, IS_SPARK_REMOTE
+from ibis.conftest import IS_SPARK_REMOTE
 
 pd = pytest.importorskip("pandas")
 pa = pytest.importorskip("pyarrow")
@@ -494,11 +494,6 @@ def test_to_pyarrow_decimal(backend, dtype, pyarrow_dtype):
 )
 @pytest.mark.notyet(["clickhouse"], raises=Exception)
 @pytest.mark.notyet(["mssql"], raises=PyDeltaTableError)
-@pytest.mark.xfail_version(
-    pyspark=["pyspark<4"],
-    condition=CI and IS_SPARK_REMOTE,
-    reason="not supported until pyspark 4",
-)
 def test_roundtrip_delta(backend, con, alltypes, tmp_path, monkeypatch):
     if con.name == "pyspark":
         pytest.importorskip("delta")