Test Explorer Support via VSTest

paket.dependencies

@@ -44,6 +44,7 @@ nuget Microsoft.NET.Test.Sdk >= 17.4
 nuget Dotnet.ReproducibleBuilds copy_local:true
 
 nuget Ionide.KeepAChangelog.Tasks copy_local: true
+nuget Expecto ~> 10
 nuget Expecto.Diff
 nuget YoloDev.Expecto.TestSdk
 nuget AltCover
@@ -58,3 +59,8 @@ nuget CommunityToolkit.HighPerformance
 nuget System.Security.Cryptography.Pkcs
 nuget System.Net.Http
 nuget System.Text.RegularExpressions
+
+
+## Test Explorer
+nuget Microsoft.TestPlatform.TranslationLayer
+nuget Microsoft.TestPlatform.ObjectModel

paket.lock

@@ -36,7 +36,7 @@ NUGET
       Serilog (>= 3.1.1)
     DiffPlex (1.7.2) - restriction: || (== net8.0) (== net9.0) (&& (== netstandard2.0) (>= net6.0)) (&& (== netstandard2.1) (>= net6.0))
     Dotnet.ReproducibleBuilds (1.2.25) - copy_local: true
-    Expecto (10.2.1) - restriction: || (== net8.0) (== net9.0) (&& (== netstandard2.0) (>= net6.0)) (&& (== netstandard2.1) (>= net6.0))
+    Expecto (10.2.3)
       FSharp.Core (>= 7.0.200) - restriction: || (== net8.0) (== net9.0) (&& (== netstandard2.0) (>= net6.0)) (&& (== netstandard2.1) (>= net6.0))
       Mono.Cecil (>= 0.11.4 < 1.0) - restriction: || (== net8.0) (== net9.0) (&& (== netstandard2.0) (>= net6.0)) (&& (== netstandard2.1) (>= net6.0))
     Expecto.Diff (10.2.1)
@@ -400,6 +400,8 @@ NUGET
     Microsoft.TestPlatform.TestHost (17.12) - restriction: || (== net8.0) (== net9.0) (&& (== netstandard2.0) (>= netcoreapp3.1)) (&& (== netstandard2.1) (>= netcoreapp3.1))
       Microsoft.TestPlatform.ObjectModel (>= 17.12) - restriction: || (== net8.0) (== net9.0) (&& (== netstandard2.0) (>= netcoreapp3.1)) (&& (== netstandard2.1) (>= netcoreapp3.1))
       Newtonsoft.Json (>= 13.0.1) - restriction: || (== net8.0) (== net9.0) (&& (== netstandard2.0) (>= netcoreapp3.1)) (&& (== netstandard2.1) (>= netcoreapp3.1))
+    Microsoft.TestPlatform.TranslationLayer (17.13)
+      NETStandard.Library (>= 2.0)
     Microsoft.VisualStudio.SolutionPersistence (1.0.28) - restriction: || (== net8.0) (== net9.0) (&& (== netstandard2.0) (>= net8.0)) (&& (== netstandard2.1) (>= net8.0))
     Microsoft.VisualStudio.Threading (17.12.19)
       Microsoft.Bcl.AsyncInterfaces (>= 6.0) - restriction: || (&& (== net8.0) (>= net472)) (&& (== net8.0) (< net6.0)) (&& (== net9.0) (>= net472)) (&& (== net9.0) (< net6.0)) (== netstandard2.0) (== netstandard2.1)
@@ -423,6 +425,8 @@ NUGET
       Microsoft.VisualStudio.Validation (>= 17.8.8)
       System.IO.Pipelines (>= 8.0)
       System.Runtime.CompilerServices.Unsafe (>= 6.0) - restriction: || (&& (== net8.0) (< net6.0)) (&& (== net8.0) (< netstandard2.1)) (&& (== net9.0) (< net6.0)) (&& (== net9.0) (< netstandard2.1)) (== netstandard2.0) (== netstandard2.1)
+    NETStandard.Library (2.0.3)
+      Microsoft.NETCore.Platforms (>= 1.1)
     Newtonsoft.Json (13.0.3)
     Nuget.Frameworks (6.12.1) - copy_local: false
     OpenTelemetry (1.10)

src/FsAutoComplete.Core/FsAutoComplete.Core.fsproj

@@ -3,7 +3,8 @@
     <TargetFrameworks>net8.0</TargetFrameworks>
     <TargetFrameworks Condition="'$(BuildNet9)' == 'true'">net8.0;net9.0</TargetFrameworks>
     <IsPackable>false</IsPackable>
-    <NoWarn>$(NoWarn);FS0057</NoWarn> <!-- Allows using experimental FCS APIs -->
+    <NoWarn>$(NoWarn);FS0057</NoWarn>
+    <!-- Allows using experimental FCS APIs -->
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\FsAutoComplete.Logging\FsAutoComplete.Logging.fsproj" />
@@ -18,6 +19,8 @@
     <Compile Include="Debug.fs" />
     <Compile Include="Utils.fsi" />
     <Compile Include="Utils.fs" />
+    <Compile Include="VSTestWrapper.fs" />
+    <Compile Include="TestServer.fs" />
     <Compile Include="TestAdapter.fs" />
     <Compile Include="DotnetNewTemplate.fs" />
     <Compile Include="DotnetCli.fs" />
@@ -64,4 +67,4 @@
     <Compile Include="Commands.fs" />
   </ItemGroup>
   <Import Project="..\..\.paket\Paket.Restore.targets" />
-</Project>
+</Project>

src/FsAutoComplete.Core/TestServer.fs

@@ -0,0 +1,88 @@
+namespace FsAutoComplete.TestServer
+
+open System
+
+type TestFileRange = { StartLine: int; EndLine: int }
+
+type TestItem =
+  {
+    FullName: string
+    DisplayName: string
+    /// Identifies the test adapter that ran the tests
+    /// Example: executor://xunit/VsTestRunner2/netcoreapp
+    /// Used for determining the test library, which effects how tests names are broken down
+    ExecutorUri: string
+    ProjectFilePath: string
+    TargetFramework: string
+    CodeFilePath: string option
+    CodeLocationRange: TestFileRange option
+  }
+
+module TestItem =
+  let ofVsTestCase
+    (projFilePath: string)
+    (targetFramework: string)
+    (testCase: Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase)
+    : TestItem =
+    { FullName = testCase.FullyQualifiedName
+      DisplayName = testCase.DisplayName
+      ExecutorUri = testCase.ExecutorUri |> string
+      ProjectFilePath = projFilePath
+      TargetFramework = targetFramework
+      CodeFilePath = Some testCase.CodeFilePath
+      CodeLocationRange =
+        Some
+          { StartLine = testCase.LineNumber
+            EndLine = testCase.LineNumber } }
+
+  let tryTestCaseToDTO
+    (projectLookup: string -> Ionide.ProjInfo.Types.ProjectOptions option)
+    (testCase: Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase)
+    : TestItem option =
+    match projectLookup testCase.Source with
+    | None -> None // this should never happen. We pass VsTest the list of executables to test, so all the possible sources should be known to us
+    | Some project -> ofVsTestCase project.ProjectFileName project.TargetFramework testCase |> Some
+
+[<RequireQualifiedAccess>]
+type TestOutcome =
+  | Failed = 0
+  | Passed = 1
+  | Skipped = 2
+  | None = 3
+  | NotFound = 4
+
+module TestOutcome =
+  type VSTestOutcome = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome
+
+  let ofVSTestOutcome (vsTestOutcome: VSTestOutcome) =
+    match vsTestOutcome with
+    | VSTestOutcome.Passed -> TestOutcome.Passed
+    | VSTestOutcome.Failed -> TestOutcome.Failed
+    | VSTestOutcome.Skipped -> TestOutcome.Skipped
+    | VSTestOutcome.NotFound -> TestOutcome.NotFound
+    | VSTestOutcome.None -> TestOutcome.None
+    | _ -> TestOutcome.None
+
+type TestResult =
+  { TestItem: TestItem
+    Outcome: TestOutcome
+    ErrorMessage: string option
+    ErrorStackTrace: string option
+    AdditionalOutput: string option
+    Duration: TimeSpan }
+
+module TestResult =
+  type VSTestResult = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult
+
+  let ofVsTestResult (projFilePath: string) (targetFramework: string) (vsTestResult: VSTestResult) : TestResult =
+    let stringToOption (text: string) = if String.IsNullOrEmpty(text) then None else Some text
+
+    { Outcome = TestOutcome.ofVSTestOutcome vsTestResult.Outcome
+      ErrorMessage = vsTestResult.ErrorMessage |> stringToOption
+      ErrorStackTrace = vsTestResult.ErrorStackTrace |> stringToOption
+      AdditionalOutput =
+        match vsTestResult.Messages |> Seq.toList with
+        | [] -> None
+        | messages -> messages |> List.map _.Text |> String.concat Environment.NewLine |> Some
+      Duration = vsTestResult.Duration
+      TestItem = TestItem.ofVsTestCase projFilePath targetFramework vsTestResult.TestCase }

src/FsAutoComplete.Core/VSTestWrapper.fs

@@ -0,0 +1,191 @@
+namespace FsAutoComplete.TestServer
+
+open System
+
+module VSTestWrapper =
+  open Microsoft.TestPlatform.VsTestConsole.TranslationLayer
+  open Microsoft.VisualStudio.TestPlatform.ObjectModel
+  open Microsoft.VisualStudio.TestPlatform.ObjectModel.Client
+  open Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging
+  open Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces
+
+  type TestProjectDll = string
+
+  type TestDiscoveryUpdate =
+    | Progress of TestCase list
+    | LogMessage of TestMessageLevel * string
+
+  type private TestDiscoveryHandler(notifyDiscoveryProgress: TestDiscoveryUpdate -> unit) =
+
+    member val DiscoveredTests: TestCase ResizeArray = ResizeArray() with get, set
+
+    interface ITestDiscoveryEventsHandler with
+      member this.HandleDiscoveredTests(discoveredTestCases: System.Collections.Generic.IEnumerable<TestCase>) : unit =
+        if (not << isNull) discoveredTestCases then
+          this.DiscoveredTests.AddRange(discoveredTestCases)
+          notifyDiscoveryProgress (discoveredTestCases |> List.ofSeq |> Progress)
+
+      member this.HandleDiscoveryComplete
+        (_totalTests: int64, lastChunk: System.Collections.Generic.IEnumerable<TestCase>, _isAborted: bool)
+        : unit =
+        if (not << isNull) lastChunk then
+          this.DiscoveredTests.AddRange(lastChunk)
+          notifyDiscoveryProgress (lastChunk |> List.ofSeq |> Progress)
+
+      member this.HandleLogMessage(level: TestMessageLevel, message: string) : unit =
+        notifyDiscoveryProgress (LogMessage(level, message))
+
+      member this.HandleRawMessage(_rawMessage: string) : unit = ()
+
+  type ProcessId = int
+  type DidDebuggerAttach = bool
+
+  type TestRunUpdate =
+    | Progress of TestRunChangedEventArgs
+    | LogMessage of TestMessageLevel * string
+
+  type private TestRunHandler(notifyTestRunProgress: TestRunUpdate -> unit) =
+
+    member val TestResults: TestResult ResizeArray = ResizeArray() with get, set
+
+    interface ITestRunEventsHandler with
+      member _.HandleLogMessage(level: TestMessageLevel, message: string) : unit =
+        notifyTestRunProgress (LogMessage(level, message))
+
+      member _.HandleRawMessage(_rawMessage: string) : unit = ()
+
+      member this.HandleTestRunComplete
+        (
+          _testRunCompleteArgs: TestRunCompleteEventArgs,
+          lastChunkArgs: TestRunChangedEventArgs,
+          _runContextAttachments: System.Collections.Generic.ICollection<AttachmentSet>,
+          _executorUris: System.Collections.Generic.ICollection<string>
+        ) : unit =
+        if ((not << isNull) lastChunkArgs && (not << isNull) lastChunkArgs.NewTestResults) then
+          this.TestResults.AddRange(lastChunkArgs.NewTestResults)
+          notifyTestRunProgress (Progress lastChunkArgs)
+
+      member this.HandleTestRunStatsChange(testRunChangedArgs: TestRunChangedEventArgs) : unit =
+        if
+          ((not << isNull) testRunChangedArgs
+           && (not << isNull) testRunChangedArgs.NewTestResults)
+        then
+          this.TestResults.AddRange(testRunChangedArgs.NewTestResults)
+          notifyTestRunProgress (Progress testRunChangedArgs)
+
+      member _.LaunchProcessWithDebuggerAttached(_testProcessStartInfo: TestProcessStartInfo) : int =
+        raise (System.NotImplementedException())
+
+  type private TestHostLauncher(isDebug: bool, onAttachDebugger: ProcessId -> DidDebuggerAttach) =
+    // IMPORTANT: RunTestsWithCustomTestHost says it takes an ITestHostLauncher, but it actually calls a method that is only available on ITestHostLauncher3
+
+    interface ITestHostLauncher3 with
+      member _.IsDebug: bool = isDebug
+
+      member _.LaunchTestHost(_defaultTestHostStartInfo: TestProcessStartInfo) : int = raise (NotImplementedException())
+
+      member _.LaunchTestHost
+        (_defaultTestHostStartInfo: TestProcessStartInfo, _cancellationToken: Threading.CancellationToken)
+        : int =
+        raise (NotImplementedException())
+
+      member _.AttachDebuggerToProcess
+        (attachDebuggerInfo: AttachDebuggerInfo, _cancellationToken: Threading.CancellationToken)
+        : bool =
+        onAttachDebugger attachDebuggerInfo.ProcessId
+
+      member _.AttachDebuggerToProcess(pid: int) : bool = onAttachDebugger pid
+
+      member _.AttachDebuggerToProcess(pid: int, _cancellationToken: Threading.CancellationToken) : bool =
+        onAttachDebugger pid
+
+
+  module TestPlatformOptions =
+    let withTestCaseFilter (options: TestPlatformOptions) filterExpression = options.TestCaseFilter <- filterExpression
+
+  module private RunSettings =
+    let defaultRunSettings =
+      "<RunSettings>
+    <RunConfiguration>
+        <DesignMode>False</DesignMode>
+    </RunConfiguration>
+</RunSettings>"
+
+  let discoverTestsAsync
+    (vstestPath: string)
+    (onDiscoveryProgress: TestDiscoveryUpdate -> unit)
+    (sources: TestProjectDll list)
+    : Async<TestCase list> =
+    async {
+      let consoleParams = ConsoleParameters()
+      let vstest = new VsTestConsoleWrapper(vstestPath, consoleParams)
+      let discoveryHandler = TestDiscoveryHandler(onDiscoveryProgress)
+
+      use! _onCancel = Async.OnCancel(fun () -> vstest.CancelDiscovery())
+
+      vstest.DiscoverTests(sources, null, discoveryHandler)
+      return discoveryHandler.DiscoveredTests |> List.ofSeq
+    }
+
+  /// onAttachDebugger assumes that the debugger is attached when the method returns. The test project will continue execution as soon as attachDebugger returns
+  let runTestsAsync
+    (vstestPath: string)
+    (onTestRunProgress: TestRunUpdate -> unit)
+    (onAttachDebugger: ProcessId -> DidDebuggerAttach)
+    (sources: TestProjectDll list)
+    (testCaseFilter: string option)
+    (shouldDebug: bool)
+    : Async<TestResult list> =
+    async {
+      let consoleParams = ConsoleParameters()
+      let vstest = new VsTestConsoleWrapper(vstestPath, consoleParams)
+      let runHandler = TestRunHandler(onTestRunProgress)
+      let runSettings = RunSettings.defaultRunSettings
+
+      let options = new TestPlatformOptions()
+      testCaseFilter |> Option.iter (TestPlatformOptions.withTestCaseFilter options)
+
+      use! _cancel = Async.OnCancel(fun () -> vstest.CancelTestRun())
+
+      if shouldDebug then
+        let hostLauncher = TestHostLauncher(shouldDebug, onAttachDebugger)
+        vstest.RunTestsWithCustomTestHost(sources, runSettings, options, runHandler, hostLauncher)
+      else
+        vstest.RunTests(sources, runSettings, options, runHandler)
+
+      return runHandler.TestResults |> List.ofSeq
+    }
+
+  open System.IO
+
+  let tryFindVsTestFromDotnetRoot (dotnetRoot: string) (workspaceRoot: string option) : Result<FileInfo, string> =
+    let cwd =
+      defaultArg workspaceRoot System.Environment.CurrentDirectory |> DirectoryInfo
+
+    let dotnetBinary =
+      if dotnetRoot |> Directory.Exists then
+        if
+          System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(
+            System.Runtime.InteropServices.OSPlatform.Windows
+          )
+        then
+          FileInfo(Path.Combine(dotnetRoot, "dotnet.exe"))
+        else
+          FileInfo(Path.Combine(dotnetRoot, "dotnet"))
+      else
+        dotnetRoot |> FileInfo
+
+    match Ionide.ProjInfo.SdkDiscovery.versionAt cwd dotnetBinary with
+    | Ok sdkVersion ->
+      let sdks = Ionide.ProjInfo.SdkDiscovery.sdks dotnetBinary
+
+      match sdks |> Array.tryFind (fun sdk -> sdk.Version = sdkVersion) with
+      | Some sdk ->
+        let vstestBinary = Path.Combine(sdk.Path.FullName, "vstest.console.dll") |> FileInfo
+
+        if vstestBinary.Exists then
+          Ok vstestBinary
+        else
+          Error $"Found the correct dotnet sdk, but vstest was not at the expected sub-path: {vstestBinary.FullName}"
+      | None -> Error $"Couldn't find the install location for dotnet sdk version: {sdkVersion}"
+    | Error _ -> Error $"Couldn't identify the dotnet version for working directory: {cwd.FullName}"

src/FsAutoComplete.Core/paket.references

@@ -17,3 +17,6 @@ FSharp.Analyzers.SDK
 Ionide.Analyzers
 FSharp.Analyzers.Build
 Serilog.Extensions.Logging
+
+Microsoft.TestPlatform.ObjectModel
+Microsoft.TestPlatform.TranslationLayer

src/FsAutoComplete/CommandResponse.fs

@@ -702,3 +702,17 @@ module CommandResponse =
           PrecedingNonPipeExprLine = pnp })
 
     serialize { Kind = "pipelineHint"; Data = ctn }
+
+
+
+  type DiscoverTestsResponse = TestServer.TestItem list
+
+  let discoverTests (serialize: Serializer) (content: DiscoverTestsResponse) =
+    serialize
+      { Kind = "discoverTests"
+        Data = content }
+
+  let runTests (serialize: Serializer) (content: TestServer.TestResult list) =
+    serialize
+      { Kind = "runTests"
+        Data = content |> Array.ofList }

src/FsAutoComplete/CommandResponse.fsi

@@ -261,3 +261,8 @@ module CommandResponse =
   val compile: serialize: Serializer -> errors: #FSharpDiagnostic array * code: int -> string
   val fsharpLiterate: serialize: Serializer -> content: string -> string
   val pipelineHint: serialize: Serializer -> content: (int * int option * string list)[] -> string
+
+  type DiscoverTestsResponse = TestServer.TestItem list
+
+  val discoverTests: serialize: Serializer -> content: DiscoverTestsResponse -> string
+  val runTests: serialize: Serializer -> content: TestServer.TestResult list -> string