Replace Newtonsoft.Json with System.Text.Json in tests and samples

- Migrate JsonBulkCopyTest and JsonStreamTest to System.Text.Json
- Add JsonTestHelper with DeepEquals supporting net8.0+ and net9.0+
- Rewrite SqlExceptionTest to serialize key properties via STJ
- Remove JSONSerializationTest (tested Newtonsoft-specific ISerializable)
- Delete legacy AzureKeyVaultProviderLegacyExample_2_0.cs (marked TODO)
- Remove Newtonsoft.Json from all csproj files and Directory.Packages.props

Tracks: #4322

Author: paulmedynski

Directory.Packages.props

@@ -71,7 +71,6 @@
   <ItemGroup>
     <PackageVersion Include="Azure.Core" Version="1.51.1" />
     <PackageVersion Include="Microsoft.Identity.Client" Version="4.83.0" />
-    <PackageVersion Include="Newtonsoft.Json" Version="13.0.4" />
   </ItemGroup>
 
   <ItemGroup Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net9.0'))">

doc/samples/AzureKeyVaultProviderLegacyExample_2_0.cs

@@ -23,7 +23,6 @@
     <PackageReference Include="Azure.Identity" />
     <PackageReference Include="Azure.Core" />
     <PackageReference Include="Microsoft.Identity.Client" />
-    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="System.Configuration.ConfigurationManager" />
   </ItemGroup>
 

doc/samples/Microsoft.Data.SqlClient.Samples.csproj

@@ -67,7 +67,6 @@
     <PackageReference Include="Microsoft.Extensions.Hosting" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="Microsoft.SqlServer.Types" />
-    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="System.Buffers" />
     <PackageReference Include="System.Configuration.ConfigurationManager" />
     <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" />
@@ -92,7 +91,6 @@
     <PackageReference Include="Microsoft.Extensions.Hosting" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="Microsoft.SqlServer.Types" />
-    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="System.Configuration.ConfigurationManager" />
     <PackageReference Include="System.Data.Odbc" />
     <PackageReference Include="System.Security.Cryptography.Pkcs" />

src/Microsoft.Data.SqlClient/tests/FunctionalTests/Microsoft.Data.SqlClient.FunctionalTests.csproj

@@ -5,7 +5,7 @@
 using System;
 using System.IO;
 using System.Runtime.Serialization.Formatters.Binary;
-using Newtonsoft.Json;
+using System.Text.Json;
 using Xunit;
 
 namespace Microsoft.Data.SqlClient.Tests
@@ -18,46 +18,27 @@ public class SqlExceptionTest
         public void SerializationTest()
         {
             SqlException e = CreateException();
-            string json = JsonConvert.SerializeObject(e);
 
-            var settings = new JsonSerializerSettings();
-            var sqlEx = JsonConvert.DeserializeObject<SqlException>(json, settings);
-
-            Assert.Equal(e.ClientConnectionId, sqlEx.ClientConnectionId);
-            Assert.Equal(e.StackTrace, sqlEx.StackTrace);
-        }
+            // Serialize the properties we want to validate round-trip through JSON.
+            // SqlException cannot be directly serialized by System.Text.Json because
+            // Exception.TargetSite (MethodBase) is not supported.
+            string json = JsonSerializer.Serialize(new
+            {
+                e.Message,
+                ClientConnectionId = e.ClientConnectionId.ToString(),
+                e.Number,
+                e.Class,
+                e.State,
+            });
 
-        [Fact]
-        public void JSONSerializationTest()
-        {
-            string clientConnectionId = "90cdab4d-2145-4c24-a354-c8ccff903542";
-            string json = @"{"
-                        + @"""ClassName"":""Microsoft.Data.SqlClient.SqlException"","
-                        + @"""Message"":""A network-related or instance-specific error occurred while establishing a connection to SQL Server. The server was not found or was not accessible. Verify that the instance name is correct and that SQL Server is configured to allow remote connections. (provider: TCP Provider, error: 40 - Could not open a connection to SQL Server)"","
-                        + @"""Data"":{""HelpLink.ProdName"":""Microsoft SQL Server"","
-                        + @"""HelpLink.EvtSrc"":""MSSQLServer"","
-                        + @"""HelpLink.EvtID"":""0"","
-                        + @"""HelpLink.BaseHelpUrl"":""http://go.microsoft.com/fwlink"","
-                        + @"""HelpLink.LinkId"":""20476"","
-                        + @"""SqlError 1"":""Microsoft.Data.SqlClient.SqlError: A network-related or instance-specific error occurred while establishing a connection to SQL Server. The server was not found or was not accessible. Verify that the instance name is correct and that SQL Server is configured to allow remote connections. (provider: TCP Provider, error: 40 - Could not open a connection to SQL Server)"","
-                        + @"""$type"":""System.Collections.ListDictionaryInternal, System.Private.CoreLib""},"
-                        + @"""InnerException"":null,"
-                        + @"""HelpURL"":null,"
-                        + @"""StackTraceString"":""   at Microsoft.Data.SqlClient.SqlInternalConnectionTds..ctor(DbConnectionPoolIdentity identity, SqlConnectionOptions connectionOptions, SqlCredential credential, Object providerInfo, String newPassword, SecureString newSecurePassword, Boolean redirectedUserInstance, SqlConnectionOptions userConnectionOptions, SessionData reconnectSessionData, Boolean applyTransientFaultHandling, String accessToken)\\n   at Microsoft.Data.SqlClient.SqlConnectionFactory.CreateConnection(SqlConnectionOptions options, DbConnectionPoolKey poolKey, Object poolGroupProviderInfo, DbConnectionPool pool, DbConnection owningConnection, SqlConnectionOptions userOptions)\\n   at System.Data.ProviderBase.DbConnectionFactory.CreatePooledConnection(DbConnectionPool pool, DbConnection owningObject, SqlConnectionOptions options, DbConnectionPoolKey poolKey, SqlConnectionOptions userOptions)\\n   at System.Data.ProviderBase.DbConnectionPool.CreateObject(DbConnection owningObject, SqlConnectionOptions userOptions, DbConnectionInternal oldConnection)\\n   at System.Data.ProviderBase.DbConnectionPool.UserCreateRequest(DbConnection owningObject, SqlConnectionOptions userOptions, DbConnectionInternal oldConnection)\\n   at System.Data.ProviderBase.DbConnectionPool.TryGetConnection(DbConnection owningObject, UInt32 waitForMultipleObjectsTimeout, Boolean allowCreate, Boolean onlyOneCheckConnection, SqlConnectionOptions userOptions, DbConnectionInternal& connection)\\n   at System.Data.ProviderBase.DbConnectionPool.WaitForPendingOpen()\\n"","
-                        + @"""RemoteStackTraceString"":null,"
-                        + @"""RemoteStackIndex"":0,"
-                        + @"""ExceptionMethod"":null,"
-                        + @"""HResult"":-2146232060,"
-                        + @"""Source"":""Core .Net SqlClient Data Provider"","
-                        + @"""WatsonBuckets"":null,"
-                        + @"""Errors"":null,"
-                        + @"""ClientConnectionId"":""90cdab4d-2145-4c24-a354-c8ccff903542"""
-                        + @"}";
+            using JsonDocument doc = JsonDocument.Parse(json);
+            JsonElement root = doc.RootElement;
 
-            var settings = new JsonSerializerSettings();
-            var sqlEx = JsonConvert.DeserializeObject<SqlException>(json, settings);
-            Assert.IsType<SqlException>(sqlEx);
-            Assert.Equal(clientConnectionId, sqlEx.ClientConnectionId.ToString());
+            Assert.Equal(e.Message, root.GetProperty("Message").GetString());
+            Assert.Equal(e.ClientConnectionId.ToString(), root.GetProperty("ClientConnectionId").GetString());
+            Assert.Equal(e.Number, root.GetProperty("Number").GetInt32());
+            Assert.Equal(e.Class, root.GetProperty("Class").GetByte());
+            Assert.Equal(e.State, root.GetProperty("State").GetByte());
         }
 
         private static SqlException CreateException()

src/Microsoft.Data.SqlClient/tests/FunctionalTests/SqlExceptionTest.cs

@@ -199,6 +199,7 @@
     <Compile Include="SQL\JsonTest\JsonBulkCopyTest.cs" />
     <Compile Include="SQL\JsonTest\JsonStreamTest.cs" />
     <Compile Include="SQL\JsonTest\JsonTest.cs" />
+    <Compile Include="SQL\JsonTest\JsonTestHelper.cs" />
     <Compile Include="SQL\KerberosTests\KerberosTest.cs" />
     <Compile Include="SQL\KerberosTests\KerberosTicketManager\KerberosTicketManager.cs" />
     <Compile Include="SQL\LocalDBTest\LocalDBTest.cs" />
@@ -362,7 +363,6 @@
     <PackageReference Include="Microsoft.DotNet.XUnitExtensions" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="Microsoft.SqlServer.Types" />
-    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="System.Configuration.ConfigurationManager" />
     <PackageReference Include="System.Runtime.InteropServices.RuntimeInformation" />
     <PackageReference Include="System.Security.Cryptography.Pkcs" />
@@ -390,7 +390,6 @@
     <PackageReference Include="Microsoft.DotNet.XUnitExtensions" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="Microsoft.SqlServer.Types" />
-    <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="System.Configuration.ConfigurationManager" />
     <PackageReference Include="System.Security.Cryptography.Pkcs" />
     <PackageReference Include="System.ServiceProcess.ServiceController" />

src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj

@@ -6,12 +6,10 @@
 using System.Collections.Generic;
 using System.Data;
 using System.IO;
+using System.Text.Json;
 using System.Threading.Tasks;
-using Newtonsoft.Json.Linq;
-using Newtonsoft.Json;
-using Xunit.Abstractions;
 using Xunit;
-using System.Collections;
+using Xunit.Abstractions;
 
 namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.JsonTest
 {
@@ -67,23 +65,17 @@ private void GenerateJsonFile(int noOfRecords, string filename)
                 });
             }
 
-            string json = JsonConvert.SerializeObject(records, Formatting.Indented);
+            string json = JsonSerializer.Serialize(records, new JsonSerializerOptions { WriteIndented = true });
             File.WriteAllText(filename, json);
             Assert.True(File.Exists(filename));
             _output.WriteLine("Generated JSON file " + filename);
         }
 
         private void CompareJsonFiles()
         {
-            using (var stream1 = File.OpenText(_generatedJsonFile))
-            using (var stream2 = File.OpenText(_outputFile))
-            using (var reader1 = new JsonTextReader(stream1))
-            using (var reader2 = new JsonTextReader(stream2))
-            {
-                var jToken1 = JToken.ReadFrom(reader1);
-                var jToken2 = JToken.ReadFrom(reader2);
-                Assert.True(JToken.DeepEquals(jToken1, jToken2));
-            }
+            using JsonDocument doc1 = JsonDocument.Parse(File.ReadAllText(_generatedJsonFile));
+            using JsonDocument doc2 = JsonDocument.Parse(File.ReadAllText(_outputFile));
+            Assert.True(JsonTestHelper.JsonDeepEquals(doc1.RootElement, doc2.RootElement));
         }
 
         private void PrintJsonDataToFileAndCompare(SqlConnection connection)

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonBulkCopyTest.cs

@@ -8,11 +8,11 @@
 using System.Data;
 using System.Linq;
 using System.Text;
+using System.Text.Json;
 using System.Threading.Tasks;
-using Newtonsoft.Json;
 using Xunit;
 using Xunit.Abstractions;
-using Newtonsoft.Json.Linq;
+using Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.JsonTest;
 using Microsoft.Data.SqlClient.Tests.Common.Fixtures.DatabaseObjects;
 
 
@@ -51,23 +51,17 @@ private void GenerateJsonFile(int noOfRecords, string filename)
                 });
             }
 
-            string json = JsonConvert.SerializeObject(records, Formatting.Indented);
+            string json = JsonSerializer.Serialize(records, new JsonSerializerOptions { WriteIndented = true });
             File.WriteAllText(filename, json);
             Assert.True(File.Exists(filename));
             _output.WriteLine("Generated JSON file "+filename);
         }
 
         private void CompareJsonFiles()
         {
-            using (var stream1 = File.OpenText(_jsonFile))
-            using (var stream2 = File.OpenText(_outputFile))
-            using (var reader1 = new JsonTextReader(stream1))
-            using (var reader2 = new JsonTextReader(stream2))
-            {
-                var jToken1 = JToken.ReadFrom(reader1);
-                var jToken2 = JToken.ReadFrom(reader2);
-                Assert.True(JToken.DeepEquals(jToken1, jToken2));
-            }
+            using JsonDocument doc1 = JsonDocument.Parse(File.ReadAllText(_jsonFile));
+            using JsonDocument doc2 = JsonDocument.Parse(File.ReadAllText(_outputFile));
+            Assert.True(JsonTestHelper.JsonDeepEquals(doc1.RootElement, doc2.RootElement));
         }
 
         private void PrintJsonDataToFile(SqlConnection connection, string tableName)

src/Microsoft.Data.SqlClient/tests/ManualTests/SQL/JsonTest/JsonStreamTest.cs

@@ -0,0 +1,81 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
+namespace Microsoft.Data.SqlClient.ManualTesting.Tests.SQL.JsonTest
+{
+    internal static class JsonTestHelper
+    {
+        // Test data is Array → Object → Value (3 levels). Use 64 as a safe ceiling.
+        private const int MaxDepth = 64;
+
+        /// <summary>
+        /// Performs a deep structural comparison of two <see cref="JsonElement"/> values.
+        /// On .NET 9+ this delegates to <see cref="JsonNode.DeepEquals"/>; on earlier
+        /// runtimes it uses a recursive comparison over the element trees.
+        /// </summary>
+        internal static bool JsonDeepEquals(JsonElement a, JsonElement b)
+        {
+#if NET9_0_OR_GREATER
+            return JsonNode.DeepEquals(
+                JsonNode.Parse(a.GetRawText()),
+                JsonNode.Parse(b.GetRawText()));
+#else
+            return DeepEqualsCore(a, b, depth: 0);
+#endif
+        }
+
+        private static bool DeepEqualsCore(JsonElement a, JsonElement b, int depth)
+        {
+            if (depth > MaxDepth)
+            {
+                throw new InvalidOperationException($"JSON comparison exceeded maximum depth of {MaxDepth}.");
+            }
+
+            if (a.ValueKind != b.ValueKind)
+            {
+                return false;
+            }
+
+            switch (a.ValueKind)
+            {
+                case JsonValueKind.Object:
+                    int countA = a.EnumerateObject().Count();
+                    int countB = b.EnumerateObject().Count();
+                    if (countA != countB)
+                    {
+                        return false;
+                    }
+                    foreach (JsonProperty prop in a.EnumerateObject())
+                    {
+                        if (!b.TryGetProperty(prop.Name, out JsonElement bValue) ||
+                            !DeepEqualsCore(prop.Value, bValue, depth + 1))
+                        {
+                            return false;
+                        }
+                    }
+                    return true;
+
+                case JsonValueKind.Array:
+                    JsonElement.ArrayEnumerator arrA = a.EnumerateArray();
+                    JsonElement.ArrayEnumerator arrB = b.EnumerateArray();
+                    while (arrA.MoveNext())
+                    {
+                        if (!arrB.MoveNext() || !DeepEqualsCore(arrA.Current, arrB.Current, depth + 1))
+                        {
+                            return false;
+                        }
+                    }
+                    return !arrB.MoveNext();
+
+                default:
+                    return a.GetRawText() == b.GetRawText();
+            }
+        }
+    }
+}