Update spark connect dependencies to be 4.0

Author: allisonport-db

build.sbt

@@ -335,7 +335,7 @@ lazy val connectClient = (project in file("spark-connect/client"))
     ),
     (Test / javaOptions) += s"-Ddelta.test.home=" + file(".").getAbsoluteFile.getParentFile,
     (Test / resourceGenerators) += Def.task {
-      val location = url("https://dist.apache.org/repos/dist/release/spark/spark-4.0.0-preview1/spark-4.0.0-preview1-bin-hadoop3.tgz")
+      val location = url("https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc4-bin/spark-4.0.0-bin-hadoop3.tgz")
       val destDir = (Test / resourceManaged).value / "spark"
       if (!destDir.exists()) {
         IO.createDirectory(destDir)
@@ -371,7 +371,7 @@ lazy val connectClient = (project in file("spark-connect/client"))
       }
     }.taskValue,
     (Test / resourceGenerators) += Def.task {
-      val src = url("https://repository.apache.org/content/groups/public/org/apache/spark/spark-connect_2.13/4.0.0-preview1/spark-connect_2.13-4.0.0-preview1.jar")
+      val src = url("https://repository.apache.org/content/repositories/orgapachespark-1480/org/apache/spark/spark-connect_2.13/4.0.0/spark-connect_2.13-4.0.0.jar")
       val dest = (Test / resourceManaged).value / "spark-connect.jar"
       if (!dest.exists()) {
         src #> dest !;

python/run-tests.py

@@ -204,7 +204,7 @@ def run_delta_connect_codegen_python(root_dir):
         # TODO: In the future, find a way to get these
         # packages locally instead of downloading from Maven.
         delta_connect_packages = ["com.google.protobuf:protobuf-java:3.25.1",
-                                  "org.apache.spark:spark-connect_2.13:4.0.0-preview1",
+                                  "org.apache.spark:spark-connect_2.13:4.0.0",
                                   get_local_package("delta-connect-server", use_spark_master)]
 
         test(root_dir, path.join("delta", "connect"), delta_connect_packages)