Merge pull request #1381 from microsoft/fix1307

Add msbuild item to opt 3rd party libraries into app-local deployments

Author: AArnott

docfx/docs/3rdPartyMetadata.md

@@ -0,0 +1,55 @@
+# 3rd party metadata
+
+CsWin32 comes with dependencies on Windows metadata for the SDK and WDK, allowing C# programs to generate interop code for Windows applications.
+But the general transformation from metadata to C# code may be applied to other metadata inputs, allowing you to generate similar metadata for 3rd party native libraries and use CsWin32 to generate C# interop APIs for it.
+
+## Constructing metadata for other libraries
+
+Constructing metadata is outside the scope of this document.
+However you may find [the win32metadata architecture](https://github.com/microsoft/win32metadata/blob/main/docs/architecture.md) document instructive.
+
+## Hooking metadata into CsWin32
+
+Metadata is fed into CsWin32 through MSBuild items.
+
+Item Type | Purpose
+--|--
+`ProjectionMetadataWinmd` | Path to the .winmd file.
+`ProjectionDocs` | Path to an optional msgpack data file that contains API-level documentation.
+`AppLocalAllowedLibraries` | The filename (including extension) of a native library that is allowed to ship in the app directory (as opposed to only %windir%\system32).
+
+## Packaging up metadata
+
+Build a NuGet package with the following layout:
+
+```
+buildTransitive\
+   YourPackageId.props
+   yournativelib.winmd
+runtimes\
+   win-x86\
+      yournativelib.dll
+   win-x64\
+      yournativelib.dll
+   win-arm64\
+      yournativelib.dll
+   ...
+```
+
+Your package metadata may want to express a dependency on the Microsoft.Windows.CsWin32 package.
+
+The `YourPackageId.props` file should include the msbuild items above, as appropriate.
+For example:
+
+```xml
+<Project>
+  <ItemGroup>
+    <ProjectionMetadataWinmd Include="$(MSBuildThisFileDirectory)yournativelib.winmd" />
+    <AppLocalAllowedLibraries Include="yournativelib.dll" />
+  </ItemGroup>
+</Project>
+```
+
+## Consuming your package
+
+A project can reference your NuGet package to get both the native dll deployed with their app and the C# interop APIs generated as they require through NativeMethods.txt using CsWin32, just like they can for Win32 APIs.

docfx/docs/toc.yml

@@ -1,3 +1,5 @@
 items:
 - href: features.md
 - href: getting-started.md
+- href: 3rdPartyMetadata.md
+

src/Microsoft.Windows.CsWin32/Generator.Extern.cs

@@ -147,19 +147,6 @@ internal void RequestExternMethod(MethodDefinitionHandle methodDefinitionHandle)
         this.volatileCode.GenerateMethod(methodDefinitionHandle, () => this.DeclareExternMethod(methodDefinitionHandle));
     }
 
-    private static bool IsLibraryAllowedAppLocal(string libraryName)
-    {
-        for (int i = 0; i < AppLocalLibraries.Length; i++)
-        {
-            if (string.Equals(libraryName, AppLocalLibraries[i], StringComparison.OrdinalIgnoreCase))
-            {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
     private string GetMethodNamespace(MethodDefinition methodDef) => this.Reader.GetString(this.Reader.GetTypeDefinition(methodDef.GetDeclaringType()).Namespace);
 
     private void DeclareExternMethod(MethodDefinitionHandle methodDefinitionHandle)
@@ -233,7 +220,7 @@ AttributeListSyntax CreateDllImportAttributeList()
                 if (this.generateDefaultDllImportSearchPathsAttribute)
                 {
                     result = result.AddAttributes(
-                        IsLibraryAllowedAppLocal(moduleName)
+                        this.AppLocalLibraries.Contains(moduleName)
                             ? DefaultDllImportSearchPathsAllowAppDirAttribute
                             : DefaultDllImportSearchPathsAttribute);
                 }

src/Microsoft.Windows.CsWin32/Generator.Invariants.cs

@@ -144,10 +144,13 @@ public partial class Generator
 ");
 
     /// <summary>
-    /// The set of libraries that are expected to be allowed next to an application instead of being required to load from System32.
+    /// The initial set of libraries that are expected to be allowed next to an application instead of being required to load from System32.
     /// </summary>
+    /// <remarks>
+    /// This list is combined with an MSBuild item list so that 3rd party metadata can document app-local DLLs.
+    /// </remarks>
     /// <see href="https://docs.microsoft.com/en-us/windows/win32/debug/dbghelp-versions" />
-    private static readonly string[] AppLocalLibraries = new[] { "DbgHelp.dll", "SymSrv.dll", "SrcSrv.dll" };
+    private static readonly string[] BuiltInAppLocalLibraries = ["DbgHelp.dll", "SymSrv.dll", "SrcSrv.dll"];
 
     // [DefaultDllImportSearchPaths(DllImportSearchPath.System32)]
     private static readonly AttributeSyntax DefaultDllImportSearchPathsAttribute =

src/Microsoft.Windows.CsWin32/Generator.cs

@@ -74,10 +74,11 @@ static Generator()
     /// </summary>
     /// <param name="metadataLibraryPath">The path to the winmd metadata to generate APIs from.</param>
     /// <param name="docs">The API docs to include in the generated code.</param>
+    /// <param name="additionalAppLocalLibraries">The library file names (e.g. some.dll) that should be allowed as app-local.</param>
     /// <param name="options">Options that influence the result of generation.</param>
     /// <param name="compilation">The compilation that the generated code will be added to.</param>
     /// <param name="parseOptions">The parse options that will be used for the generated code.</param>
-    public Generator(string metadataLibraryPath, Docs? docs, GeneratorOptions options, CSharpCompilation? compilation = null, CSharpParseOptions? parseOptions = null)
+    public Generator(string metadataLibraryPath, Docs? docs, IEnumerable<string> additionalAppLocalLibraries, GeneratorOptions options, CSharpCompilation? compilation = null, CSharpParseOptions? parseOptions = null)
     {
         if (options is null)
         {
@@ -90,6 +91,9 @@ public Generator(string metadataLibraryPath, Docs? docs, GeneratorOptions option
 
         this.ApiDocs = docs;
 
+        this.AppLocalLibraries = new(BuiltInAppLocalLibraries, StringComparer.OrdinalIgnoreCase);
+        this.AppLocalLibraries.UnionWith(additionalAppLocalLibraries);
+
         this.options = options;
         this.options.Validate();
         this.compilation = compilation;
@@ -291,6 +295,8 @@ internal Generator MainGenerator
     /// </summary>
     internal Context DefaultContext => new() { AllowMarshaling = this.options.AllowMarshaling };
 
+    private HashSet<string> AppLocalLibraries { get; }
+
     private bool WideCharOnly => this.options.WideCharOnly;
 
     private string Namespace => this.MetadataIndex.CommonNamespace;

src/Microsoft.Windows.CsWin32/SourceGenerator.cs

@@ -152,6 +152,13 @@ public class SourceGenerator : ISourceGenerator
         '\u200B', // ZERO WIDTH SPACE (U+200B)
     };
 
+    private static readonly JsonSerializerOptions JsonOptions = new JsonSerializerOptions
+    {
+        AllowTrailingCommas = true,
+        ReadCommentHandling = JsonCommentHandling.Skip,
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+    };
+
     /// <inheritdoc/>
     public void Initialize(GeneratorInitializationContext context)
     {
@@ -173,12 +180,7 @@ public void Execute(GeneratorExecutionContext context)
             string optionsJson = nativeMethodsJsonFile.GetText(context.CancellationToken)!.ToString();
             try
             {
-                options = JsonSerializer.Deserialize<GeneratorOptions>(optionsJson, new JsonSerializerOptions
-                {
-                    AllowTrailingCommas = true,
-                    ReadCommentHandling = JsonCommentHandling.Skip,
-                    PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
-                });
+                options = JsonSerializer.Deserialize<GeneratorOptions>(optionsJson, JsonOptions);
             }
             catch (JsonException ex)
             {
@@ -210,8 +212,9 @@ public void Execute(GeneratorExecutionContext context)
             context.ReportDiagnostic(Diagnostic.Create(MissingRecommendedReference, location: null, "System.Memory"));
         }
 
+        IEnumerable<string> appLocalLibraries = CollectAppLocalAllowedLibraries(context);
         Docs? docs = ParseDocs(context);
-        Generator[] generators = CollectMetadataPaths(context).Select(path => new Generator(path, docs, options, compilation, parseOptions)).ToArray();
+        Generator[] generators = CollectMetadataPaths(context).Select(path => new Generator(path, docs, appLocalLibraries, options, compilation, parseOptions)).ToArray();
         if (TryFindNonUniqueValue(generators, g => g.InputAssemblyName, StringComparer.OrdinalIgnoreCase, out (Generator Item, string Value) nonUniqueGenerator))
         {
             context.ReportDiagnostic(Diagnostic.Create(NonUniqueMetadataInputs, null, nonUniqueGenerator.Value));
@@ -389,6 +392,17 @@ private static IReadOnlyList<string> CollectMetadataPaths(GeneratorExecutionCont
         return metadataBasePaths;
     }
 
+    private static IEnumerable<string> CollectAppLocalAllowedLibraries(GeneratorExecutionContext context)
+    {
+        if (!context.AnalyzerConfigOptions.GlobalOptions.TryGetValue("build_property.CsWin32AppLocalAllowedLibraries", out string? delimitedAppLocalLibraryPaths) ||
+            string.IsNullOrWhiteSpace(delimitedAppLocalLibraryPaths))
+        {
+            return Array.Empty<string>();
+        }
+
+        return delimitedAppLocalLibraryPaths.Split('|').Select(Path.GetFileName);
+    }
+
     private static Docs? ParseDocs(GeneratorExecutionContext context)
     {
         Docs? docs = null;

src/Microsoft.Windows.CsWin32/build/Microsoft.Windows.CsWin32.props

@@ -14,13 +14,15 @@
     <!-- Provide the path to the winmds used as input into the analyzer. -->
     <CompilerVisibleProperty Include="CsWin32InputMetadataPaths" />
     <CompilerVisibleProperty Include="CsWin32InputDocPaths" />
+    <CompilerVisibleProperty Include="CsWin32AppLocalAllowedLibraries" />
   </ItemGroup>
 
   <Target Name="AssembleCsWin32InputPaths" BeforeTargets="GenerateMSBuildEditorConfigFileCore">
     <!-- Roslyn only allows source generators to see msbuild properties, to lift msbuild items into semicolon-delimited properties. -->
     <PropertyGroup>
       <CsWin32InputMetadataPaths>@(ProjectionMetadataWinmd->'%(FullPath)','|')</CsWin32InputMetadataPaths>
       <CsWin32InputDocPaths>@(ProjectionDocs->'%(FullPath)','|')</CsWin32InputDocPaths>
+      <CsWin32AppLocalAllowedLibraries>@(AppLocalAllowedLibraries->'%(FullPath)','|')</CsWin32AppLocalAllowedLibraries>
     </PropertyGroup>
   </Target>
 </Project>

test/Microsoft.Windows.CsWin32.Tests/GeneratorTestBase.cs

@@ -377,7 +377,7 @@ protected SuperGenerator CreateGenerator(GeneratorOptions? options = null, CShar
         => this.CreateSuperGenerator(DefaultMetadataPaths, options, compilation, includeDocs);
 
     protected SuperGenerator CreateSuperGenerator(string[] metadataPaths, GeneratorOptions? options = null, CSharpCompilation? compilation = null, bool includeDocs = false) =>
-        SuperGenerator.Combine(metadataPaths.Select(path => new Generator(path, includeDocs ? Docs.Get(ApiDocsPath) : null, options ?? DefaultTestGeneratorOptions, compilation ?? this.compilation, this.parseOptions)));
+        SuperGenerator.Combine(metadataPaths.Select(path => new Generator(path, includeDocs ? Docs.Get(ApiDocsPath) : null, [], options ?? DefaultTestGeneratorOptions, compilation ?? this.compilation, this.parseOptions)));
 
     private static ImmutableArray<Diagnostic> FilterDiagnostics(ImmutableArray<Diagnostic> diagnostics) => diagnostics.Where(d => d.Severity > DiagnosticSeverity.Hidden && d.Descriptor.Id != "CS1701").ToImmutableArray();