Fix: use MsBuild Locator rather than embed

Directory.Packages.props

@@ -22,13 +22,29 @@
     <PackageVersion Include="YamlDotNet" Version="16.3.0" />
   </ItemGroup>
 
-  <ItemGroup>
-    <PackageVersion Include="Microsoft.Build" Version="[17.11.48]" Condition="'$(TargetFramework)' == 'net8.0'" />
-    <PackageVersion Include="Microsoft.Build" Version="[17.14.28]" Condition="'$(TargetFramework)' == 'net9.0'" />
-    <PackageVersion Include="Microsoft.Build" Version="18.0.2"     Condition="'$(TargetFramework)' != 'net8.0' AND '$(TargetFramework)' != 'net9.0'" />
+  <ItemGroup Condition="'$(TargetFramework)' == 'net8.0'">
+    <PackageVersion Include="Microsoft.Build" Version="[17.11.48]" />
+    <PackageVersion Include="Microsoft.Build.Framework" Version="[17.11.48]" />
+    <PackageVersion Include="Microsoft.Build.Tasks.Core" Version="[17.11.48]" />
+    <PackageVersion Include="Microsoft.Build.Utilities.Core" Version="[17.11.48]" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(TargetFramework)' == 'net9.0'">
+    <PackageVersion Include="Microsoft.Build" Version="[17.14.28]" />
+    <PackageVersion Include="Microsoft.Build.Framework" Version="[17.14.28]" />
+    <PackageVersion Include="Microsoft.Build.Tasks.Core" Version="[17.14.28]" />
+    <PackageVersion Include="Microsoft.Build.Utilities.Core" Version="[17.14.28]" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(TargetFramework)' != 'net8.0' AND '$(TargetFramework)' != 'net9.0'">
+    <PackageVersion Include="Microsoft.Build" Version="18.0.2" />
+    <PackageVersion Include="Microsoft.Build.Framework" Version="18.0.2" />
+    <PackageVersion Include="Microsoft.Build.Tasks.Core" Version="18.0.2" />
+    <PackageVersion Include="Microsoft.Build.Utilities.Core" Version="18.0.2" />
   </ItemGroup>
 
   <ItemGroup>
+    <PackageVersion Include="Microsoft.Build.Locator" Version="1.11.2" />
     <PackageVersion Include="Microsoft.CodeAnalysis" Version="5.0.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.Common" Version="5.0.0" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="5.0.0" />

docs/reference/docfx-cli-reference/docfx-metadata.md

@@ -108,6 +108,15 @@ Run `docfx metadata --help` or `docfx -h` to get a list of all available options
     - `true`
         - The CLR type names are used: `Int32`.
 
+- **--forceBuild**
+
+  Indicates whether to force MSBuild to run a full build prior to metadata extraction being attempted. This is useful if you have code that is produced outside of normal design time mechanisms, but will increase the time taken depending on the size of the project(s).
+
+    - not specified or `false`
+        - A build is not attempted.
+    - `true`
+        - A build is attempted.
+
 ## Examples
 
 - Generate YAML files with default config.

docs/reference/docfx-json-reference.md

@@ -480,6 +480,10 @@ Specifies whether explicit interface implementations are included in the generat
 
 Specify the name to use for the global namespace. The default value is an empty string.
 
+### `forceBuild`
+
+When enabled, will force MSBuild to run a full build prior to metadata extraction being attempted.
+
 ## File Mappings
 
 In the short-hand form, these filenames are resolved relative to the directory containing the `docfx.json` file:

schemas/docfx.schema.json

@@ -701,6 +701,11 @@
             "type": "boolean",
             "default": false,
             "description": "When enabled, use CLR type names instead of language aliases."
+          },
+          "forceBuild": {
+            "type": "boolean",
+            "default": false,
+            "description": "When enabled, will force MSBuild to run a full build prior to metadata extraction being attempted."
           }
         }
       }

src/Docfx.Dotnet/Docfx.Dotnet.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <ItemGroup>
     <EmbeddedResource Include="Resources\**" />
   </ItemGroup>
@@ -30,15 +30,15 @@
     <PackageReference Include="OneOf" />
     <PackageReference Include="OneOf.SourceGenerator" PrivateAssets="All" />
     <PackageReference Include="Markdig" />
-    <PackageReference Include="Microsoft.Build" />
-    <PackageReference Include="Microsoft.CodeAnalysis" />
-    <PackageReference Include="Microsoft.CodeAnalysis.Common" />
+    <PackageReference Include="Microsoft.Build" ExcludeAssets="runtime" PrivateAssets="all" />
+    <PackageReference Include="Microsoft.Build.Framework" ExcludeAssets="runtime" PrivateAssets="all" />
+    <PackageReference Include="Microsoft.Build.Locator" />
+    <PackageReference Include="Microsoft.Build.Tasks.Core" ExcludeAssets="runtime" PrivateAssets="all" />
+    <PackageReference Include="Microsoft.Build.Utilities.Core" ExcludeAssets="runtime" PrivateAssets="all" />
     <PackageReference Include="Microsoft.CodeAnalysis.VisualBasic" />
     <PackageReference Include="Microsoft.CodeAnalysis.VisualBasic.Workspaces" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" />
-    <PackageReference Include="Microsoft.CodeAnalysis.Workspaces.Common" />
     <PackageReference Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" />
   </ItemGroup>
-
 </Project>

src/Docfx.Dotnet/DotnetApiCatalog.Compile.cs

@@ -62,6 +62,8 @@ partial class DotnetApiCatalog
             foreach (var solution in solutionFiles.Select(s => s.NormalizedPath))
             {
                 Logger.LogInfo($"Loading solution {solution}");
+                var sf = SolutionFile.Parse(solution);
+
                 foreach (var project in SolutionFile.Parse(solution).ProjectsInOrder)
                 {
                     if (project.ProjectType is SolutionProjectType.KnownToBeMSBuildFormat &&
@@ -150,43 +152,67 @@ await LoadCompilationFromProject(project.AbsolutePath) is { } compilation)
             if (project is null)
             {
                 Logger.LogInfo($"Loading project {path}");
-                if (!config.NoRestore)
+                if (!config.NoRestore || config.ForceBuild)
                 {
                     using var process = Process.Start("dotnet", $"restore \"{path}\"");
                     await process.WaitForExitAsync();
                 }
-                project = await workspace.OpenProjectAsync(path, msbuildLogger);
-
-                foreach (var unresolvedAnalyzer in project.AnalyzerReferences.OfType<UnresolvedAnalyzerReference>())
-                {
-                    Logger.LogWarning($"There is .NET Analyzer that can't be resolved. "
-                                    + $"If this analyzer is .NET Source Generator project. "
-                                    + $"Try build with `dotnet build -c Release` command before running docfx. Path: {unresolvedAnalyzer.FullPath}",
-                                    code: WarningCodes.Metadata.FailedToResolveAnalyzer);
-                }
 
-                foreach (var analyzer in project.AnalyzerReferences.OfType<AnalyzerFileReference>())
+                if (config.ForceBuild)
                 {
-                    analyzer.AnalyzerLoadFailed += (sender, e) =>
+                    // Build using MSBuild APIs
+                    var buildParams = new Microsoft.Build.Execution.BuildParameters
                     {
-                        var analyzerName = analyzer.Display;
-                        var errorCode = e.ErrorCode;
-                        var referencedCompilerVersion = e.ReferencedCompilerVersion;
-
-                        Logger.LogWarning($"Failed to load .NET Analyzer. AnalyzerName: {analyzerName}, ErrorCode: {errorCode}, ReferencedCompilerVersion: {referencedCompilerVersion}",
-                                          code: WarningCodes.Metadata.FailedToLoadAnalyzer);
+                        Loggers = new[] { msbuildLogger }
                     };
 
+                    var buildRequest = new Microsoft.Build.Execution.BuildRequestData(
+                        path,
+                        msbuildProperties,
+                        null,
+                        new[] { "Build" },
+                        null);
+
+                    var buildResult = Microsoft.Build.Execution.BuildManager.DefaultBuildManager.Build(buildParams, buildRequest);
+
+                    if (buildResult.OverallResult != Microsoft.Build.Execution.BuildResultCode.Success)
+                    {
+                        Logger.LogWarning($"Build failed for project {path}");
+                    }
                 }
+
+                project = await workspace.OpenProjectAsync(path, msbuildLogger);
             }
 
-            if (!project.SupportsCompilation)
+            var compilation = await project.GetCompilationAsync();
+            if (compilation == null)
             {
                 Logger.LogInfo($"Skip unsupported project {project.FilePath}.");
                 return null;
             }
 
-            return await project.GetCompilationAsync();
+            foreach (var unresolvedAnalyzer in project.AnalyzerReferences.OfType<UnresolvedAnalyzerReference>())
+            {
+                Logger.LogWarning($"There is .NET Analyzer that can't be resolved. "
+                                  + $"If this analyzer is .NET Source Generator project. "
+                                  + $"Try build with `dotnet build -c Release` command before running docfx. Path: {unresolvedAnalyzer.FullPath}",
+                    code: WarningCodes.Metadata.FailedToResolveAnalyzer);
+            }
+
+            foreach (var analyzer in project.AnalyzerReferences.OfType<AnalyzerFileReference>())
+            {
+                analyzer.AnalyzerLoadFailed += (sender, e) =>
+                {
+                    var analyzerName = analyzer.Display;
+                    var errorCode = e.ErrorCode;
+                    var referencedCompilerVersion = e.ReferencedCompilerVersion;
+
+                    Logger.LogWarning($"Failed to load .NET Analyzer. AnalyzerName: {analyzerName}, ErrorCode: {errorCode}, ReferencedCompilerVersion: {referencedCompilerVersion}",
+                        code: WarningCodes.Metadata.FailedToLoadAnalyzer);
+                };
+            }
+
+            return compilation;
         }
     }
 }

src/Docfx.Dotnet/DotnetApiCatalog.cs

@@ -5,6 +5,7 @@
 using System.Text.Json;
 using Docfx.Common;
 using Docfx.Plugins;
+using Microsoft.Build.Locator;
 using Newtonsoft.Json.Linq;
 using YamlDotNet.Serialization;
 
@@ -65,6 +66,22 @@ internal static async Task Exec(MetadataJsonConfig config, DotnetApiOptions opti
 
             EnvironmentContext.SetBaseDirectory(configDirectory);
 
+            // this is here because heading into Compile, MSBuildLocator must have been registered as it uses MSBuild APIs
+            if (!MSBuildLocator.IsRegistered)
+            {
+                var visualStudioInstanceQueryOptions = VisualStudioInstanceQueryOptions.Default;
+                Logger.LogInfo($"Searching for MSBuild based upon: DiscoveryTypes:{visualStudioInstanceQueryOptions.DiscoveryTypes} - AllowAllDotnetLocations:{visualStudioInstanceQueryOptions.AllowAllDotnetLocations} - AllowAllRuntimeVersions:{visualStudioInstanceQueryOptions.AllowAllRuntimeVersions} - WorkingDirectory:\"{visualStudioInstanceQueryOptions.WorkingDirectory}\"");
+
+                var latestVersion = MSBuildLocator.QueryVisualStudioInstances(visualStudioInstanceQueryOptions).MaxBy(instance => instance.Version);
+                if (latestVersion == null)
+                {
+                    throw new InvalidOperationException("Failed to find a version of Visual Studio or .NET SDK installed");
+                }
+
+                MSBuildLocator.RegisterInstance(latestVersion);
+                Logger.LogInfo($"Located MSBuild for: {latestVersion.Name} - {latestVersion.Version}");
+            }
+
             foreach (var item in config)
             {
                 VisitorHelper.GlobalNamespaceId = item.GlobalNamespaceId;
@@ -143,6 +160,7 @@ private static ExtractMetadataConfig ConvertConfig(MetadataJsonItemConfig config
             DisableDefaultFilter = configModel?.DisableDefaultFilter ?? false,
             DisableGitFeatures = configModel?.DisableGitFeatures ?? false,
             NoRestore = configModel?.NoRestore ?? false,
+            ForceBuild = configModel?.ForceBuild ?? false,
             CategoryLayout = configModel?.CategoryLayout ?? default,
             NamespaceLayout = configModel?.NamespaceLayout ?? default,
             MemberLayout = configModel?.MemberLayout ?? default,

src/Docfx.Dotnet/ManagedReference/ExtractMetadataConfig.cs

@@ -45,4 +45,5 @@ internal class ExtractMetadataConfig
 
     public static bool UseClrTypeNames { get; set; }
 
+    public bool ForceBuild { get; init; }
 }

src/Docfx.Dotnet/MetadataJsonConfig.cs

@@ -265,6 +265,13 @@ internal class MetadataJsonItemConfig
     [JsonProperty("useClrTypeNames")]
     [JsonPropertyName("useClrTypeNames")]
     public bool UseClrTypeNames { get; init; }
+
+    /// <summary>
+    /// Forces building the project before metadata extraction.
+    /// </summary>
+    [JsonProperty("forceBuild")]
+    [JsonPropertyName("forceBuild")]
+    public bool ForceBuild { get; set; }
 }
 
 /// <summary>

src/docfx/Models/MetadataCommand.cs

@@ -32,6 +32,7 @@ private static void MergeOptionsToConfig(MetadataCommandOptions options, DocfxCo
             item.NamespaceLayout = options.NamespaceLayout ?? item.NamespaceLayout;
             item.MemberLayout = options.MemberLayout ?? item.MemberLayout;
             item.OutputFormat = options.OutputFormat ?? item.OutputFormat;
+            item.ForceBuild |= options.ForceBuild;
 
             if (!string.IsNullOrEmpty(options.FilterConfigFile))
             {

src/docfx/Models/MetadataCommandOptions.cs

@@ -60,4 +60,8 @@ internal class MetadataCommandOptions : LogOptions
     [Description("Determines the member page layout.")]
     [CommandOption("--memberLayout")]
     public MemberLayout? MemberLayout { get; set; }
+
+    [Description("Forces a Full Build to be run before metadata is gathered.")]
+    [CommandOption("--forceBuild")]
+    public bool ForceBuild { get; set; }
 }