Could not run UDF with EntityFrameworkCore

[hsfert]

Is there a way to run UDF in the worker nodes when using EntityFrameworkCore? I am running the application on Windows 7 and at .Net Core 3.1 and my code looks something as follows:

static void Main(string[] args)
{
    int N = 4;
    var output = MyUDF(73546);
    GenericRow[] rows = new GenericRow[N];
    for(int i = 0; i < N; i++)
    {
        rows[i] = new GenericRow(new object[]{73546});
    }
    StructType structType = new StructType(
        new StructField[]
        {
            new StructField("id", new IntegerType())
    });
    var spark = SparkSession.Builder().GetOrCreate();
    DateFrame df = spark.CreateDataFrame(rows, structType);
    df.Show();
    df.CreateOrReplaceTempView("tmp_table");
    spark.Udf().Register<int?, long>("MyUDF", MyUDF);
    var df2 = spark.Sql("SELECT MyUDF(id) FROM temp_table);
    df2.Show();
    spark.Stop();
}

static long MyUDF(int? id)
{
    using(var content = new DataEntity())
    {
        var output = content.DataObjs.FirstOrDefault(obj => obj.id == (id ?? 0));
        if (output != null)
        {
             return output.secondaryId;
        }
    }
    return 0;
}

When I deploy the application via DotnetRunner, the first call MyUDF(73546) runs okay. But when it is running through the worker node, there is an error:

Microsoft.Data.SqlClient is not supported on this platform.; at Microsoft.Data.SqlClient.SqlConnection..ctor(String connectionString)
    at Microsoft.EntityFrameworkCore.SqlServer.Storage.Internal.SqlServerConnection.CreateDbConnection()

When I searched this error on the web, it seems that deploying the application on the azure service will encountering the same issue and they need to create function.deps.json to fix that.

[imback82]

@hsfert I am not familiar with entity framework, but can you make sure you have the same setting for worker nodes as the driver node? If you have a repro code I can test on my box, that will be helpful as well. (I am assuming DataEntity is your code that utilizes the entity framework)

[hsfert]

I can reproduce the error without an existing SQL server as follows. I have just installed two packages Microsoft.Spark and System.Data.SqlClient. I am deploying the application in the local machine using --master local so I assume that the driver node and the worker node have the same setting.

using System;
using System.Collections.Generic;
using Microsoft.Spark.Sql;
using Microsoft.Spark.Sql.Types;
using static Microsoft.Spark.Sql.Functions;
using System.Data.SqlClient;

namespace SparkTesting
{
    class Program
    {
        static void Main(string[] args)
        {
            int N = 4;
            var output = MyUDF(73546);
            GenericRow[] rows = new GenericRow[N];
            for (int i = 0; i < N; i++)
            {
                rows[i] = new GenericRow(new object[] { 73546 });
            }
            StructType structType = new StructType(
                new StructField[]
                {
            new StructField("id", new IntegerType())
            });
            var spark = SparkSession.Builder().GetOrCreate();
            var df = spark.CreateDataFrame(rows, structType);
            df.Show();
            df.CreateOrReplaceTempView("tmp_table");
            spark.Udf().Register<int?, long>("MyUDF", MyUDF);
            var df2 = spark.Sql("SELECT MyUDF(id) FROM tmp_table");
            df2.Show();
            spark.Stop();
        }

        static long MyUDF(int? id)
        {
            using(var content = new SqlConnection(""))
            {

            }
            return 0;
        }
    }
}

The SparkTesting.deps.json is as follows:

{
  "runtimeTarget": {
    "name": ".NETCoreApp,Version=v3.1",
    "signature": ""
  },
  "compilationOptions": {},
  "targets": {
    ".NETCoreApp,Version=v3.1": {
      "SparkTesting/1.0.0": {
        "dependencies": {
          "Microsoft.Spark": "0.10.0",
          "System.Data.SqlClient": "4.8.1"
        },
        "runtime": {
          "SparkTesting.dll": {}
        }
      },
      "Apache.Arrow/0.14.1": {
        "dependencies": {
          "System.Buffers": "4.5.0",
          "System.Memory": "4.5.2",
          "System.Runtime.CompilerServices.Unsafe": "4.5.2",
          "System.Threading.Tasks.Extensions": "4.5.2"
        },
        "runtime": {
          "lib/netcoreapp2.1/Apache.Arrow.dll": {
            "assemblyVersion": "0.14.1.0",
            "fileVersion": "0.14.1.0"
          }
        }
      },
      "Microsoft.CSharp/4.5.0": {},
      "Microsoft.NETCore.Platforms/3.1.0": {},
      "Microsoft.Spark/0.10.0": {
        "dependencies": {
          "Apache.Arrow": "0.14.1",
          "Microsoft.CSharp": "4.5.0",
          "Newtonsoft.Json": "11.0.2",
          "Razorvine.Pyrolite": "4.26.0",
          "System.Memory": "4.5.2"
        },
        "runtime": {
          "lib/netstandard2.1/Microsoft.Spark.dll": {
            "assemblyVersion": "0.10.0.0",
            "fileVersion": "0.1000.20.12602"
          }
        }
      },
      "Microsoft.Win32.Registry/4.7.0": {
        "dependencies": {
          "System.Security.AccessControl": "4.7.0",
          "System.Security.Principal.Windows": "4.7.0"
        }
      },
      "Newtonsoft.Json/11.0.2": {
        "runtime": {
          "lib/netstandard2.0/Newtonsoft.Json.dll": {
            "assemblyVersion": "11.0.0.0",
            "fileVersion": "11.0.2.21924"
          }
        }
      },
      "Razorvine.Pyrolite/4.26.0": {
        "dependencies": {
          "Razorvine.Serpent": "1.29.0",
          "System.Memory": "4.5.2"
        },
        "runtime": {
          "lib/netstandard2.0/Razorvine.Pyrolite.dll": {
            "assemblyVersion": "4.26.0.0",
            "fileVersion": "4.26.0.0"
          }
        }
      },
      "Razorvine.Serpent/1.29.0": {
        "runtime": {
          "lib/netstandard2.0/Razorvine.Serpent.dll": {
            "assemblyVersion": "1.29.0.0",
            "fileVersion": "1.29.0.0"
          }
        }
      },
      "runtime.native.System.Data.SqlClient.sni/4.7.0": {
        "dependencies": {
          "runtime.win-arm64.runtime.native.System.Data.SqlClient.sni": "4.4.0",
          "runtime.win-x64.runtime.native.System.Data.SqlClient.sni": "4.4.0",
          "runtime.win-x86.runtime.native.System.Data.SqlClient.sni": "4.4.0"
        }
      },
      "runtime.win-arm64.runtime.native.System.Data.SqlClient.sni/4.4.0": {
        "runtimeTargets": {
          "runtimes/win-arm64/native/sni.dll": {
            "rid": "win-arm64",
            "assetType": "native",
            "fileVersion": "4.6.25512.1"
          }
        }
      },
      "runtime.win-x64.runtime.native.System.Data.SqlClient.sni/4.4.0": {
        "runtimeTargets": {
          "runtimes/win-x64/native/sni.dll": {
            "rid": "win-x64",
            "assetType": "native",
            "fileVersion": "4.6.25512.1"
          }
        }
      },
      "runtime.win-x86.runtime.native.System.Data.SqlClient.sni/4.4.0": {
        "runtimeTargets": {
          "runtimes/win-x86/native/sni.dll": {
            "rid": "win-x86",
            "assetType": "native",
            "fileVersion": "4.6.25512.1"
          }
        }
      },
      "System.Buffers/4.5.0": {},
      "System.Data.SqlClient/4.8.1": {
        "dependencies": {
          "Microsoft.Win32.Registry": "4.7.0",
          "System.Security.Principal.Windows": "4.7.0",
          "runtime.native.System.Data.SqlClient.sni": "4.7.0"
        },
        "runtime": {
          "lib/netcoreapp2.1/System.Data.SqlClient.dll": {
            "assemblyVersion": "4.6.1.1",
            "fileVersion": "4.700.20.6702"
          }
        },
        "runtimeTargets": {
          "runtimes/unix/lib/netcoreapp2.1/System.Data.SqlClient.dll": {
            "rid": "unix",
            "assetType": "runtime",
            "assemblyVersion": "4.6.1.1",
            "fileVersion": "4.700.20.6702"
          },
          "runtimes/win/lib/netcoreapp2.1/System.Data.SqlClient.dll": {
            "rid": "win",
            "assetType": "runtime",
            "assemblyVersion": "4.6.1.1",
            "fileVersion": "4.700.20.6702"
          }
        }
      },
      "System.Memory/4.5.2": {},
      "System.Runtime.CompilerServices.Unsafe/4.5.2": {},
      "System.Security.AccessControl/4.7.0": {
        "dependencies": {
          "Microsoft.NETCore.Platforms": "3.1.0",
          "System.Security.Principal.Windows": "4.7.0"
        }
      },
      "System.Security.Principal.Windows/4.7.0": {},
      "System.Threading.Tasks.Extensions/4.5.2": {}
    }
  },
  "libraries": {
    "SparkTesting/1.0.0": {
      "type": "project",
      "serviceable": false,
      "sha512": ""
    },
    "Apache.Arrow/0.14.1": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-no9kV8l0u+oMm+eCH43Vhcs2V7jjzYqcBdEtcQlRWlKwMrJrPNAErVFmALYbn+X+t3d6JwwfSl3vNM2R6yRIzQ==",
      "path": "apache.arrow/0.14.1",
      "hashPath": "apache.arrow.0.14.1.nupkg.sha512"
    },
    "Microsoft.CSharp/4.5.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-kaj6Wb4qoMuH3HySFJhxwQfe8R/sJsNJnANrvv8WdFPMoNbKY5htfNscv+LHCu5ipz+49m2e+WQXpLXr9XYemQ==",
      "path": "microsoft.csharp/4.5.0",
      "hashPath": "microsoft.csharp.4.5.0.nupkg.sha512"
    },
    "Microsoft.NETCore.Platforms/3.1.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-z7aeg8oHln2CuNulfhiLYxCVMPEwBl3rzicjvIX+4sUuCwvXw5oXQEtbiU2c0z4qYL5L3Kmx0mMA/+t/SbY67w==",
      "path": "microsoft.netcore.platforms/3.1.0",
      "hashPath": "microsoft.netcore.platforms.3.1.0.nupkg.sha512"
    },
    "Microsoft.Spark/0.10.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-+bJG0D4LjYOfn1wOO4YboDMfBiCnsl/sjfeVaLqIzPO6t+dZQRgNqmbViwFHWRmVsMB40BA7NhAIasXCJBsp1w==",
      "path": "microsoft.spark/0.10.0",
      "hashPath": "microsoft.spark.0.10.0.nupkg.sha512"
    },
    "Microsoft.Win32.Registry/4.7.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-KSrRMb5vNi0CWSGG1++id2ZOs/1QhRqROt+qgbEAdQuGjGrFcl4AOl4/exGPUYz2wUnU42nvJqon1T3U0kPXLA==",
      "path": "microsoft.win32.registry/4.7.0",
      "hashPath": "microsoft.win32.registry.4.7.0.nupkg.sha512"
    },
    "Newtonsoft.Json/11.0.2": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-IvJe1pj7JHEsP8B8J8DwlMEx8UInrs/x+9oVY+oCD13jpLu4JbJU2WCIsMRn5C4yW9+DgkaO8uiVE5VHKjpmdQ==",
      "path": "newtonsoft.json/11.0.2",
      "hashPath": "newtonsoft.json.11.0.2.nupkg.sha512"
    },
    "Razorvine.Pyrolite/4.26.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-FUFWXqrit2HSY6EJnPLCGQGPUepcnP0ZPyjkccxfwyq+D9YFB32msgBHRnh7L+YIVC46LFsVAULe+CzFd+VSkg==",
      "path": "razorvine.pyrolite/4.26.0",
      "hashPath": "razorvine.pyrolite.4.26.0.nupkg.sha512"
    },
    "Razorvine.Serpent/1.29.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-uux2fp2C1uiyLptmtsUab9bb0TNuJI1vhSOv7XlFBCD7gN9MaJjYiT+6SSpCVnuEZWl2ccjeaz187zivIpI6IQ==",
      "path": "razorvine.serpent/1.29.0",
      "hashPath": "razorvine.serpent.1.29.0.nupkg.sha512"
    },
    "runtime.native.System.Data.SqlClient.sni/4.7.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-9kyFSIdN3T0qjDQ2R0HRXYIhS3l5psBzQi6qqhdLz+SzFyEy4sVxNOke+yyYv8Cu8rPER12c3RDjLT8wF3WBYQ==",
      "path": "runtime.native.system.data.sqlclient.sni/4.7.0",
      "hashPath": "runtime.native.system.data.sqlclient.sni.4.7.0.nupkg.sha512"
    },
    "runtime.win-arm64.runtime.native.System.Data.SqlClient.sni/4.4.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-LbrynESTp3bm5O/+jGL8v0Qg5SJlTV08lpIpFesXjF6uGNMWqFnUQbYBJwZTeua6E/Y7FIM1C54Ey1btLWupdg==",
      "path": "runtime.win-arm64.runtime.native.system.data.sqlclient.sni/4.4.0",
      "hashPath": "runtime.win-arm64.runtime.native.system.data.sqlclient.sni.4.4.0.nupkg.sha512"
    },
    "runtime.win-x64.runtime.native.System.Data.SqlClient.sni/4.4.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-38ugOfkYJqJoX9g6EYRlZB5U2ZJH51UP8ptxZgdpS07FgOEToV+lS11ouNK2PM12Pr6X/PpT5jK82G3DwH/SxQ==",
      "path": "runtime.win-x64.runtime.native.system.data.sqlclient.sni/4.4.0",
      "hashPath": "runtime.win-x64.runtime.native.system.data.sqlclient.sni.4.4.0.nupkg.sha512"
    },
    "runtime.win-x86.runtime.native.System.Data.SqlClient.sni/4.4.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-YhEdSQUsTx+C8m8Bw7ar5/VesXvCFMItyZF7G1AUY+OM0VPZUOeAVpJ4Wl6fydBGUYZxojTDR3I6Bj/+BPkJNA==",
      "path": "runtime.win-x86.runtime.native.system.data.sqlclient.sni/4.4.0",
      "hashPath": "runtime.win-x86.runtime.native.system.data.sqlclient.sni.4.4.0.nupkg.sha512"
    },
    "System.Buffers/4.5.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-pL2ChpaRRWI/p4LXyy4RgeWlYF2sgfj/pnVMvBqwNFr5cXg7CXNnWZWxrOONLg8VGdFB8oB+EG2Qw4MLgTOe+A==",
      "path": "system.buffers/4.5.0",
      "hashPath": "system.buffers.4.5.0.nupkg.sha512"
    },
    "System.Data.SqlClient/4.8.1": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-HKLykcv6eZLbLnSMnlQ6Os4+UAmFE+AgYm92CTvJYeTOBtOYusX3qu8OoGhFrnKZax91UcLcDo5vPrqvJUTSNQ==",
      "path": "system.data.sqlclient/4.8.1",
      "hashPath": "system.data.sqlclient.4.8.1.nupkg.sha512"
    },
    "System.Memory/4.5.2": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-fvq1GNmUFwbKv+aLVYYdgu/+gc8Nu9oFujOxIjPrsf+meis9JBzTPDL6aP/eeGOz9yPj6rRLUbOjKMpsMEWpNg==",
      "path": "system.memory/4.5.2",
      "hashPath": "system.memory.4.5.2.nupkg.sha512"
    },
    "System.Runtime.CompilerServices.Unsafe/4.5.2": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-wprSFgext8cwqymChhrBLu62LMg/1u92bU+VOwyfBimSPVFXtsNqEWC92Pf9ofzJFlk4IHmJA75EDJn1b2goAQ==",
      "path": "system.runtime.compilerservices.unsafe/4.5.2",
      "hashPath": "system.runtime.compilerservices.unsafe.4.5.2.nupkg.sha512"
    },
    "System.Security.AccessControl/4.7.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-JECvTt5aFF3WT3gHpfofL2MNNP6v84sxtXxpqhLBCcDRzqsPBmHhQ6shv4DwwN2tRlzsUxtb3G9M3763rbXKDg==",
      "path": "system.security.accesscontrol/4.7.0",
      "hashPath": "system.security.accesscontrol.4.7.0.nupkg.sha512"
    },
    "System.Security.Principal.Windows/4.7.0": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-ojD0PX0XhneCsUbAZVKdb7h/70vyYMDYs85lwEI+LngEONe/17A0cFaRFqZU+sOEidcVswYWikYOQ9PPfjlbtQ==",
      "path": "system.security.principal.windows/4.7.0",
      "hashPath": "system.security.principal.windows.4.7.0.nupkg.sha512"
    },
    "System.Threading.Tasks.Extensions/4.5.2": {
      "type": "package",
      "serviceable": true,
      "sha512": "sha512-BG/TNxDFv0svAzx8OiMXDlsHfGw623BZ8tCXw4YLhDFDvDhNUEV58jKYMGRnkbJNm7c3JNNJDiN7JBMzxRBR2w==",
      "path": "system.threading.tasks.extensions/4.5.2",
      "hashPath": "system.threading.tasks.extensions.4.5.2.nupkg.sha512"
    }
  }
}

The error is:

Caused by: org.apache.spark.api.python.PythonException: System.PlatformNotSupportedException: System.Data.SqlClient is not supported on this platform.
  at System.Data.SqlClient.SqlConnection..ctor(String connectionString)
  at SparkTesting.Program.MyUDF(Nullable~` id) in C:\Users\user\source\repo\SparkTesting\SparkTesting\Program.cs:line 38
  at Microsoft.Spark.Sql.PicklingUdfWrapper~2.Execute(Int32 splitIndex, Object[] input, Int32[] argOffsets) in /_/src/csharp/Microsoft.Spark/PicklingUdfWrapper.cs:line 46
  at Microsoft.SPark.Worker.Command.PicklingSqlCommandExecutor.SingleCommandRunner.Run(Int32 splitId, Object input) in D:\a\1\s\src\csharp\Microsoft.Spark.WOrker\Command\SqlCommandExecutor.cs: line248
  at Microsoft.Spark.Worker.Command.PicklingSqlCommandExecutor.ExecuteCore(Stream inputStream, Stream outputStream, SqlCommand[] commands) in D:\a\1\s\src\csharp\Microsoft.Spark.Worker\Command\SqlCOmmandExecutor.cs:line 148

[imback82]

@hsfert sorry, I didn't get a chance to get to this today. But can you try to place .deps.json file in the directory where Microsoft.Spark.Worker.exe is located (since that's the process that loads the UDF and executes it). You may need to rename SparkTesting.deps.json accordingly.

[hsfert]

Ultimately what worked for me is that I just add those packages to the source code, call some of the libraries in the code and build the Microsoft.Spark.Worker application from it. I think it is not ideal as I might miss some of the configuration.

For the deps.json, Microsoft.Spark.Worker.exe seems to be only loading dll from Microsoft.Spark.Worker.deps.json and I will have to override it in order to work.

[rapoth]

@indy-3rdman since I believe he got this working.

[indy-3rdman]

@hsfert, I think that might be related to the EntityFramework provider for the SQL Server database engine. I've used EntityFramework to test the ForeachWriter and it worked fine for PostgreSQL and SQLite. But when testing with SQL-Server, I get a similar error message.

[imback82]

For the deps.json, Microsoft.Spark.Worker.exe seems to be only loading dll from Microsoft.Spark.Worker.deps.json and I will have to override it in order to work.

Yea, but I think this is an easier/better solution than having to build/maintain Microsofoft.Spark.Worker.exe.

[steven95421]

I'm facing the same error. With Microsoft.SqlServer.Types

[this-fifo]

Not sure if this is related but I can actually load and invoke System.Data.SqlClient with no problem inside a Databricks cluster on Azure

I do however, get this issue shortly after that I am not being able to solve right now.

I am thinking it could be related to the fact that the runtime is linux and the tcp socket to SQL Server needs to be configured differently than the default values but I am not sure how to achieve that, the exception doesn't help me either

===> 02:24:42.411 +00:00 [ftl] An error occurred starting the application
System.AggregateException: One or more errors occurred. (A network-related or instance-specific error occurred while establishing a connection to SQL Server. The server was not found or was not accessible. Verify that the instance name is correct and that SQL Server is configured to allow remote connections. (provider: TCP Provider, error: 35 - An internal exception was caught))
 ---> System.Data.SqlClient.SqlException (0x80131904): A network-related or instance-specific error occurred while establishing a connection to SQL Server. The server was not found or was not accessible. Verify that the instance name is correct and that SQL Server is configured to allow remote connections. (provider: TCP Provider, error: 35 - An internal exception was caught)
 ---> System.Net.Internals.SocketExceptionFactory+ExtendedSocketException (00000005, 0xFFFDFFFF): Name or service not known
   at System.Net.Dns.InternalGetHostByName(String hostName)
   at System.Net.Dns.GetHostAddresses(String hostNameOrAddress)
   at System.Data.SqlClient.SNI.SNITCPHandle.Connect(String serverName, Int32 port, TimeSpan timeout)
   at System.Data.SqlClient.SNI.SNITCPHandle..ctor(String serverName, Int32 port, Int64 timerExpire, Object callbackObject, Boolean parallel)
   at System.Data.SqlClient.SqlInternalConnectionTds..ctor(DbConnectionPoolIdentity identity, SqlConnectionString connectionOptions, SqlCredential credential, Object providerInfo, String newPassword, SecureString newSecurePassword, Boolean redirectedUserInstance, SqlConnectionString userConnectionOptions, SessionData reconnectSessionData, Boolean applyTransientFaultHandling, String accessToken)
   at System.Data.SqlClient.SqlConnectionFactory.CreateConnection(DbConnectionOptions options, DbConnectionPoolKey poolKey, Object poolGroupProviderInfo, DbConnectionPool pool, DbConnection owningConnection, DbConnectionOptions userOptions)
   at System.Data.ProviderBase.DbConnectionFactory.CreatePooledConnection(DbConnectionPool pool, DbConnection owningObject, DbConnectionOptions options, DbConnectionPoolKey poolKey, DbConnectionOptions userOptions)
   at System.Data.ProviderBase.DbConnectionPool.CreateObject(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection)
   at System.Data.ProviderBase.DbConnectionPool.UserCreateRequest(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection)
   at System.Data.ProviderBase.DbConnectionPool.TryGetConnection(DbConnection owningObject, UInt32 waitForMultipleObjectsTimeout, Boolean allowCreate, Boolean onlyOneCheckConnection, DbConnectionOptions userOptions, DbConnectionInternal& connection)
   at System.Data.ProviderBase.DbConnectionPool.TryGetConnection(DbConnection owningObject, TaskCompletionSource`1 retry, DbConnectionOptions userOptions, DbConnectionInternal& connection)
   at System.Data.ProviderBase.DbConnectionFactory.TryGetConnection(DbConnection owningConnection, TaskCompletionSource`1 retry, DbConnectionOptions userOptions, DbConnectionInternal oldConnection, DbConnectionInternal& connection)
   at System.Data.ProviderBase.DbConnectionInternal.TryOpenConnectionInternal(DbConnection outerConnection, DbConnectionFactory connectionFactory, TaskCompletionSource`1 retry, DbConnectionOptions userOptions)
   at System.Data.ProviderBase.DbConnectionClosed.TryOpenConnection(DbConnection outerConnection, DbConnectionFactory connectionFactory, TaskCompletionSource`1 retry, DbConnectionOptions userOptions)
   at System.Data.SqlClient.SqlConnection.TryOpen(TaskCompletionSource`1 retry)
   at System.Data.SqlClient.SqlConnection.Open()

Things I already did:

• SQL Server is accessible
• Azure services can talk to each other
• Databricks Cluster IP address is whitelisted

Also mentioned here

EDIT: my issue turned out to be just the connection string 🤷‍♂️