Workaround for NUnit not respecting test filters

NUnit doesn't respect test filters when VSTest is in DesignMode, which it is by default with VsTestConsoleWrapper

ionide#1383 (comment)

Author: farlee2121

src/FsAutoComplete.Core/TestServer.fs

@@ -115,8 +115,15 @@ module VSTestWrapper =
   module TestPlatformOptions =
     let withTestCaseFilter (options: TestPlatformOptions) filterExpression = options.TestCaseFilter <- filterExpression
 
-
-  /// attachDebugger assumes that the debugger is attached when the method returns. The test project will continue execution as soon as attachDebugger returns
+  module RunSettings =
+    let defaultRunSettings =
+      "<RunSettings>
+    <RunConfiguration>
+        <DesignMode>False</DesignMode>
+    </RunConfiguration>
+</RunSettings>"
+
+  /// onAttachDebugger assumes that the debugger is attached when the method returns. The test project will continue execution as soon as attachDebugger returns
   let runTestsAsync
     (vstestPath: string)
     (onTestRunProgress: TestRunUpdate -> unit)
@@ -129,6 +136,7 @@ module VSTestWrapper =
       let consoleParams = ConsoleParameters()
       let vstest = new VsTestConsoleWrapper(vstestPath, consoleParams)
       let runHandler = TestRunHandler(onTestRunProgress)
+      let runSettings = RunSettings.defaultRunSettings
 
       let options = new TestPlatformOptions()
       testCaseFilter |> Option.iter (TestPlatformOptions.withTestCaseFilter options)
@@ -141,9 +149,9 @@ module VSTestWrapper =
 
       if shouldDebug then
         let hostLauncher = TestHostLauncher(shouldDebug, onAttachDebugger)
-        vstest.RunTestsWithCustomTestHost(sources, null, options, runHandler, hostLauncher)
+        vstest.RunTestsWithCustomTestHost(sources, runSettings, options, runHandler, hostLauncher)
       else
-        vstest.RunTests(sources, null, options, runHandler)
+        vstest.RunTests(sources, runSettings, options, runHandler)
 
       return runHandler.TestResults |> List.ofSeq
     }

test/FsAutoComplete.Tests.TestExplorer/FsAutoComplete.Tests.TestExplorer.fsproj

@@ -17,6 +17,7 @@
     <PostBuildEvent>
       dotnet build $(MSBuildProjectDirectory)/SampleTestProjects/VSTest.XUnit.Tests/
       dotnet build $(MSBuildProjectDirectory)/SampleTestProjects/VSTest.XUnit.RunResults/
+      dotnet build $(MSBuildProjectDirectory)/SampleTestProjects/VSTest.NUnit/
     </PostBuildEvent>
   </PropertyGroup>
   <Import Project="..\..\.paket\Paket.Restore.targets" />

test/FsAutoComplete.Tests.TestExplorer/SampleTestProjects/VSTest.NUnit/UnitTest1.fs

@@ -0,0 +1,12 @@
+module VSTest.NUnit
+
+open NUnit.Framework
+
+[<SetUp>]
+let Setup () = ()
+
+[<Test>]
+let Test1 () = Assert.Pass()
+
+[<Test>]
+let Test2 () = Assert.Pass()

test/FsAutoComplete.Tests.TestExplorer/SampleTestProjects/VSTest.NUnit/VSTest.NUnit.fsproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
+    <IsPackable>false</IsPackable>
+    <GenerateProgramFile>true</GenerateProgramFile>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="UnitTest1.fs"/>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="FSharp.Core" Version="9.0.303" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
+    <PackageReference Include="NUnit" Version="4.4.0" />
+    <PackageReference Include="NUnit.Analyzers" Version="4.10.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="NUnit3TestAdapter" Version="5.1.0" />
+  </ItemGroup>
+</Project>

test/FsAutoComplete.Tests.TestExplorer/SampleTestProjects/paket.dependencies

@@ -0,0 +1,2 @@
+# SampleTestProjects aren't managed with paket
+# This is here to prevent the FsAutoComplete paket from trying manage sample project dependencies

test/FsAutoComplete.Tests.TestExplorer/TestRunTests.fs

@@ -13,14 +13,14 @@ let nullAttachDebugger _ = false
 let tests =
   testList
     "VSTestWrapper Test Run"
-    [ testCaseAsync "should return an empty list if given no projects"
+    [ testCaseAsync "it should return an empty list if given no projects"
       <| async {
         let expected = []
         let! actual = VSTestWrapper.runTestsAsync vstestPath ignore nullAttachDebugger [] None false
         Expect.equal actual expected ""
       }
 
-      testCaseAsync "should be able to report basic test run outcomes"
+      testCaseAsync "it should be able to report basic test run outcomes"
       <| async {
         let expected =
           [ "Tests.My test", TestOutcome.Passed
@@ -45,7 +45,7 @@ let tests =
         Expect.equal (set actual) (set expected) ""
       }
 
-      testCaseAsync "should run only tests that match the case filter"
+      testCaseAsync "it should run only tests that match the case filter"
       <| async {
         let expected =
           [ ("Tests+Nested.Test 1", TestOutcome.Passed)
@@ -72,7 +72,36 @@ let tests =
         Expect.equal (set actual) (set expected) ""
       }
 
-      testCaseAsync "should report processIds when debugging is on"
+      testCaseAsync "it should respect test filters on NUnit projects"
+      <| async {
+        // NOTE: This is an NUnit bug. NUnit doesn't respect filters when VSTest is in Design Mode, which VsTestConsoleWrapper is by default
+        //       https://github.com/ionide/FsAutoComplete/pull/1383#issuecomment-3245590606
+        let expected = [ "VSTest.NUnit.Test1", TestOutcome.Passed ]
+
+        let sources =
+          [ Path.Combine(ResourceLocators.sampleProjectsRootDir, "VSTest.NUnit/bin/Debug/net8.0/VSTest.NUnit.dll") ]
+
+        let onTestRunProgress (progress: VSTestWrapper.TestRunUpdate) =
+          match progress with
+          | VSTestWrapper.TestRunUpdate.LogMessage message -> printfn $"Sya: {message}"
+          | _ -> ()
+
+        let! runResults =
+          VSTestWrapper.runTestsAsync
+            vstestPath
+            onTestRunProgress
+            nullAttachDebugger
+            sources
+            (Some "FullyQualifiedName~Test1")
+            false
+
+        let likenessOfTestResult (result: TestResult) = (result.TestCase.FullyQualifiedName, result.Outcome)
+        let actual = runResults |> List.map likenessOfTestResult
+
+        Expect.equal (set actual) (set expected) ""
+      }
+
+      testCaseAsync "it should report processIds when debugging is on"
       <| async {
         use tokenSource = new System.Threading.CancellationTokenSource(2000)
 
@@ -103,7 +132,7 @@ let tests =
         Expect.isSome actualProcessId "Expected runTest to report a processId"
       }
 
-      testCaseAsync "should report a processId only once per process"
+      testCaseAsync "it should report a processId only once per process"
       <| async {
         use tokenSource = new System.Threading.CancellationTokenSource(1000)