removed dependency

Author: mwojtyczka

pyproject.toml

@@ -77,7 +77,6 @@ dependencies = [
     "pytest-timeout~=2.3.1",
     "pytest-xdist~=3.5.0",
     "ruff~=0.3.4",
-    "databricks-connect~=15.4.3",
 ]
 
 # store virtual env as the child of this folder. Helps VSCode (and PyCharm) to run better

tests/integration/fixtures/test_connect.py

@@ -1,6 +1,5 @@
 import os
 from pytest import fixture
-from databricks.connect import DatabricksSession
 
 
 @fixture
@@ -26,23 +25,6 @@ def test_databricks_connect_serverless(serverless_env, ws, spark):
     assert not creator  # serverless clusters don't have assigned creator
 
 
-def test_databricks_connect_serverless_set_cluster_id(serverless_env, ws, request):
-    # get the initial spark session to retrieve serverless cluster id
-    spark_serverless = DatabricksSession.builder.serverless(True).getOrCreate()
-    cluster_id = spark_serverless.conf.get("spark.databricks.clusterUsageTags.clusterId")
-    ws.config.serverless_compute_id = cluster_id
-
-    # get a new spark session with the serverless cluster id set
-    spark_serverless_new = request.getfixturevalue("spark")
-
-    rows = spark_serverless_new.sql("SELECT 1").collect()
-    assert rows[0][0] == 1
-
-    assert spark_serverless_new.conf.get("spark.databricks.clusterUsageTags.clusterId") == cluster_id
-    creator = get_cluster_creator(spark_serverless_new, ws)
-    assert not creator  # serverless clusters don't have assigned creator
-
-
 def get_cluster_creator(spark, ws):
     """
     Get the creator of the cluster that the Spark session is connected to.