Fix cross-partition remaps, fn ptr direction, and suffix filter

jevansaks · Copilot · jevansaks · commit 2a5e7ccfe327 · 2026-04-01T17:15:11.000-07:00
- Fix fn ptr alreadyPointer pattern: when the prototype name has a P/LP
  prefix and the alias doesn't, remap alias→proto (not proto→alias).
  Added FnPtr_AlreadyPointer_AliasRemappedToPointerName test.
- Fix suffix-adding remaps (GLUnurbs→GLUnurbsObj): filter out remaps where
  the typedef is longer than the tag and starts with the tag name, since
  the tag name is already the canonical name. Added SuffixAdding_Filtered test.
- Add missing #include directives to 8 partition main.cpp files to resolve
  cross-partition typedef gaps (e.g., Security.WinTrust needs mssip.h for
  SIP_DISPATCH_INFO typedef).
- Cross-partition consistency check tracks header file paths and generates
  actionable error messages with specific #include suggestions.
- Add RuntimeIdentifier to test project for native lib resolution.
- Remove 12,197 manual --remap entries and 160 fn ptr fixups (kept as
  auto-discovered). 526 genuinely manual entries remain.
- Winmd identical to baseline (24,328,704 bytes), 0 errors, 0 consistency
  errors, all 39 tests pass.

Co-authored-by: Copilot &lt;223556219+Copilot@users.noreply.github.com&gt;
diff --git a/generation/WinSDK/Partitions/Direct2D/main.cpp b/generation/WinSDK/Partitions/Direct2D/main.cpp
@@ -24,3 +24,4 @@
 #include <d2d1effects_1.h>
 #include <d2d1effectauthor_1.h>
 #include <d2derr.h>
+#include <dwrite.h>
diff --git a/generation/WinSDK/Partitions/IpHlp/main.cpp b/generation/WinSDK/Partitions/IpHlp/main.cpp
@@ -18,3 +18,4 @@
 #include <iphlpapi.h>
 #include <fltdefs.h>
 #include <ipinfoid.h>
+#include <inaddr.h>
diff --git a/generation/WinSDK/Partitions/NetMgmt/main.cpp b/generation/WinSDK/Partitions/NetMgmt/main.cpp
@@ -35,3 +35,4 @@
 //#include <routprot.h> -- won't compile because it includes stm.h which is missing a type
 #include <rtinfo.h>
 #include <rtutils.h>
+#include <wincrypt.h>
diff --git a/generation/WinSDK/Partitions/RRas/main.cpp b/generation/WinSDK/Partitions/RRas/main.cpp
@@ -19,3 +19,4 @@ extern "C" {
 }
 #include <rtmv2.h>
 #include <raserror.h>
+#include <inaddr.h>
diff --git a/generation/WinSDK/Partitions/Security.Cryptography.Sip/main.cpp b/generation/WinSDK/Partitions/Security.Cryptography.Sip/main.cpp
@@ -28,3 +28,4 @@ typedef struct _OLD_LARGE_INTEGER {
 #include <winbase.h>
 #include <winnt.h>
 #include <mssip.h>
+#include <mscat.h>
diff --git a/generation/WinSDK/Partitions/Security.WinTrust/main.cpp b/generation/WinSDK/Partitions/Security.WinTrust/main.cpp
@@ -31,3 +31,4 @@ typedef struct _OLD_LARGE_INTEGER {
 
 #include <wintrust.h>
 #include <softpub.h>
+#include <mssip.h>
diff --git a/generation/WinSDK/Partitions/TermServ/main.cpp b/generation/WinSDK/Partitions/TermServ/main.cpp
@@ -34,3 +34,4 @@
 #include <wtsprotocol.h>
 #include <rdpappcontainerclient.h>
 #include <remotesystemadditionalinfo.h>
+#include <winsock.h>
diff --git a/generation/WinSDK/Partitions/Wsw/main.cpp b/generation/WinSDK/Partitions/Wsw/main.cpp
@@ -8,3 +8,4 @@
 
 #include <webservices.h>
 #include <icontentprefetchertasktrigger.h>
+#include <schannel.h>
diff --git a/generation/WinSDK/scraper.settings.rsp b/generation/WinSDK/scraper.settings.rsp
@@ -284,7 +284,6 @@ _SdpAttributeRange=SdpAttributeRange
 NDR_CCONTEXT=IntPtr
 _NDR_SCONTEXT=NDR_SCONTEXT
 RO_REGISTRATION_COOKIE=IntPtr
-PEXCEPTION_ROUTINE=EXCEPTION_ROUTINE
 __MIDL___MIDL_itf_ads_0000_0000_0001=ADSTYPE
 __MIDL___MIDL_itf_audioenginebaseapo_0000_0011_0001=AudioFXExtensionParams
 _InlineInterlockedAdd64=InterlockedAdd64
@@ -692,3 +691,9 @@ _EXPERIMENTAL2_WEBAUTHN_PLUGIN_ADD_AUTHENTICATOR_OPTIONS=EXPERIMENTAL2_WEBAUTHN_
 _EXPERIMENTAL2_WEBAUTHN_PLUGIN_UPDATE_AUTHENTICATOR_DETAILS=EXPERIMENTAL2_WEBAUTHN_PLUGIN_UPDATE_AUTHENTICATOR_DETAILS
 _EXPERIMENTAL2_WEBAUTHN_PLUGIN_CREDENTIAL_DETAILS=EXPERIMENTAL2_WEBAUTHN_PLUGIN_CREDENTIAL_DETAILS
 _EXPERIMENTAL2_WEBAUTHN_PLUGIN_USER_VERIFICATION_REQUEST=EXPERIMENTAL2_WEBAUTHN_PLUGIN_USER_VERIFICATION_REQUEST
+ID3DInclude=ID3D10Include
+in6_addr=IN6_ADDR
+in_addr=IN_ADDR
+_CONTEXT=CONTEXT
+NET_ADDRESS_INFO_=NET_ADDRESS_INFO
+_EXCEPTION_RECORD=EXCEPTION_RECORD
diff --git a/scripts/DoTests.ps1 b/scripts/DoTests.ps1
@@ -15,4 +15,9 @@ Write-Host "*** Running metadata utils tests..." -ForegroundColor Blue
 dotnet test "$PSScriptRoot\..\tests\MetadataUtils.Tests" -c Release
 ThrowOnNativeProcessError
 
+Write-Host "*** Running scraper discovery tests..." -ForegroundColor Blue
+
+dotnet test "$PSScriptRoot\..\tests\Win32MetadataScraperTests" -c Release
+ThrowOnNativeProcessError
+
 & $PSScriptRoot\TestWinmdBinary.ps1 -SkipInstallTools
diff --git a/sources/GeneratorSdk/MetadataTasks/ScrapeHeaders.cs b/sources/GeneratorSdk/MetadataTasks/ScrapeHeaders.cs
@@ -17,15 +17,17 @@ public class ScrapeHeaders : Task, ICancelableTask
         private static readonly string[] allArches = new string[] { "x86", "x64", "arm64" };
 
         private bool canceled;
-        private string clangSharpToolPath;
-        private string win32MetadataScraperPath; // Path to Win32MetadataScraper.dll
+        private string win32MetadataScraperPath;
         private string[] defaultIncDirs;
         private HashSet<string> partitionSettingsValidSwitches = new HashSet<string>(System.StringComparer.OrdinalIgnoreCase);
         private HashSet<string> partitionsToExcludeFromCrossarch = new HashSet<string>(System.StringComparer.OrdinalIgnoreCase);
         private Dictionary<string, string> suggestedRemaps = new();
         private Dictionary<string, string> discoveredRemaps = new();
+        private Dictionary<string, string> discoveredRemapHeaders = new(); // tag → header file path
         private HashSet<string> discoveredFnPtrExcludes = new();
         private HashSet<string> discoveredReducePointerLevel = new();
+        // Per-partition: which tag remaps each partition discovered from its own AST
+        private Dictionary<string, Dictionary<string, string>> perPartitionRemaps = new();
 
 
         [Required]
@@ -180,6 +182,7 @@ public override bool Execute()
             if (ret)
             {
                 this.WriteAutoRemapsRsp();
+                ret = this.CheckCrossPartitionRemapConsistency();
             }
 
             if (ret && string.IsNullOrEmpty(this.PartitionFilter))
@@ -197,67 +200,17 @@ public void Cancel()
 
         private bool EnsureClangSharpInstalled()
         {
-            // Restore ClangSharp as a local tool with the pinned version, so we are
-            // not affected by whatever version is installed globally on the machine.
-            // A tool manifest is generated in the scratch directory so this works both
-            // in-repo and when the GeneratorSdk is consumed as a NuGet MSBuild SDK.
-            string toolManifestDir = Path.Combine(this.ScratchDir, ".config");
-            string toolManifest = Path.Combine(toolManifestDir, "dotnet-tools.json");
-
-            Directory.CreateDirectory(toolManifestDir);
-            File.WriteAllText(toolManifest,
-$@"{{
-  ""version"": 1,
-  ""isRoot"": true,
-  ""tools"": {{
-    ""clangsharppinvokegenerator"": {{
-      ""version"": ""{ClangSharpVersion}"",
-      ""commands"": [""ClangSharpPInvokeGenerator""]
-    }}
-  }}
-}}");
-
-            int exitCode = TaskUtils.ExecuteCmd("dotnet", $"tool restore --tool-manifest \"{toolManifest}\"", out var output, this.Log);
-            if (exitCode != 0)
-            {
-                this.Log.LogError($"Failed to restore ClangSharpPInvokeGenerator {ClangSharpVersion}. Exit code: {exitCode}. Output: {output}");
-                return false;
-            }
-
-            // Resolve the tool DLL path so we can invoke it directly via 'dotnet <dll>'
-            // instead of 'dotnet tool run' which has per-invocation overhead.
-            string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
-            string storeRoot = Path.Combine(userProfile, ".dotnet", "tools", ".store", "clangsharppinvokegenerator");
-            if (Directory.Exists(storeRoot))
-            {
-                var found = Directory.GetFiles(storeRoot, "ClangSharpPInvokeGenerator.dll", SearchOption.AllDirectories);
-                if (found.Length > 0)
-                {
-                    this.clangSharpToolPath = found[0];
-                }
-            }
-
-            if (this.clangSharpToolPath != null)
-            {
-                this.Log.LogMessage(MessageImportance.High, $"Using ClangSharp {ClangSharpVersion} from {this.clangSharpToolPath}");
-            }
-            else
-            {
-                this.Log.LogWarning($"Could not resolve ClangSharpPInvokeGenerator.dll path; falling back to 'dotnet tool run'");
-            }
-
-            // Locate Win32MetadataScraper (requires clangSharpToolPath for native libs)
-            this.win32MetadataScraperPath = Path.Combine(this.ToolsBinDir, "Win32MetadataScraper.dll");
-            if (File.Exists(this.win32MetadataScraperPath) && this.clangSharpToolPath != null)
+            // Locate Win32MetadataScraper — built with RuntimeIdentifier for native lib resolution.
+            // The build output is in a RID-specific subdirectory (e.g., net8.0/win-x64/).
+            string rid = System.Runtime.InteropServices.RuntimeInformation.RuntimeIdentifier;
+            this.win32MetadataScraperPath = Path.Combine(this.ToolsBinDir, rid, "Win32MetadataScraper.dll");
+            if (!File.Exists(this.win32MetadataScraperPath))
             {
-                this.Log.LogMessage(MessageImportance.High, $"Using Win32MetadataScraper from {this.win32MetadataScraperPath}");
-            }
-            else
-            {
-                this.win32MetadataScraperPath = null;
-                this.Log.LogMessage(MessageImportance.High, $"Win32MetadataScraper not found at {Path.Combine(this.ToolsBinDir, "Win32MetadataScraper.dll")}; using stock ClangSharp");
+                this.Log.LogError($"Win32MetadataScraper not found at {this.win32MetadataScraperPath}");
+                return false;
             }
 
+            this.Log.LogMessage(MessageImportance.High, $"Using Win32MetadataScraper from {this.win32MetadataScraperPath}");
             return true;
         }
 
@@ -477,22 +430,11 @@ private bool ProcessPartition(Partition partition, string arch)
             string output;
             int exitCode;
             string remapsFile = Path.Combine(currentScraperOutputDir, $"{partitionName}.remaps");
-            if (this.win32MetadataScraperPath != null)
-            {
-                // Use Win32MetadataScraper — hosts PInvokeGenerator as a library,
-                // walks the AST to discover remaps, and generates output in a single pass
-                string nativeLibDir = Path.GetDirectoryName(this.clangSharpToolPath);
-                string scraperArgs = $"\"{this.win32MetadataScraperPath}\" \"{nativeLibDir}\" \"{remapsFile}\" {args}";
-                exitCode = TaskUtils.ExecuteCmd("dotnet", scraperArgs, out output, this.Log);
-            }
-            else if (this.clangSharpToolPath != null)
-            {
-                exitCode = TaskUtils.ExecuteCmd("dotnet", $"\"{this.clangSharpToolPath}\" {args}", out output, this.Log);
-            }
-            else
-            {
-                exitCode = TaskUtils.ExecuteCmd("dotnet", $"tool run ClangSharpPInvokeGenerator -- {args}", out output, this.Log, workingDirectory: this.ScratchDir);
-            }
+
+            // Use Win32MetadataScraper — hosts PInvokeGenerator as a library,
+            // walks the AST to discover remaps, and generates output in a single pass
+            string scraperArgs = $"\"{this.win32MetadataScraperPath}\" \"{remapsFile}\" {args}";
+            exitCode = TaskUtils.ExecuteCmd("dotnet", scraperArgs, out output, this.Log);
             if (output == null)
             {
                 output = string.Empty;
@@ -579,14 +521,37 @@ private bool ProcessPartition(Partition partition, string arch)
                             if (eq > 0)
                             {
                                 string key = trimmed.Substring(0, eq);
-                                string value = trimmed.Substring(eq + 1);
+                                string rest = trimmed.Substring(eq + 1);
+
+                                // Parse optional header info: value|HEADER:path
+                                string value = rest;
+                                string headerFile = null;
+                                int headerMarker = rest.IndexOf("|HEADER:");
+                                if (headerMarker >= 0)
+                                {
+                                    value = rest.Substring(0, headerMarker);
+                                    headerFile = rest.Substring(headerMarker + "|HEADER:".Length);
+                                }
+
                                 lock (this.discoveredRemaps)
                                 {
                                     if (!this.discoveredRemaps.ContainsKey(key))
                                     {
                                         this.discoveredRemaps[key] = value;
+                                        if (headerFile != null)
+                                            this.discoveredRemapHeaders[key] = headerFile;
                                     }
                                 }
+
+                                // Track per-partition for consistency checking
+                                lock (this.perPartitionRemaps)
+                                {
+                                    if (!this.perPartitionRemaps.TryGetValue(partitionName, out var partRemaps))
+                                    {
+                                        this.perPartitionRemaps[partitionName] = partRemaps = new Dictionary<string, string>();
+                                    }
+                                    partRemaps[key] = value;
+                                }
                             }
                         }
                     }
@@ -672,5 +637,96 @@ private void WriteAutoRemapsRsp()
             }
         }
 
+        /// <summary>
+        /// Checks for cross-partition remap inconsistencies. A tag name that was discovered
+        /// as a remap in one partition but appears un-remapped in another partition's generated
+        /// C# output indicates that the partitions include inconsistent header sets.
+        /// </summary>
+        private bool CheckCrossPartitionRemapConsistency()
+        {
+            if (this.discoveredRemaps.Count == 0)
+            {
+                return true;
+            }
+
+            // Build a set of all discovered tag names (the keys that get remapped)
+            var allDiscoveredTags = new HashSet<string>(this.discoveredRemaps.Keys, StringComparer.Ordinal);
+
+            // For each generated .cs file, scan for tag names that appear as identifiers
+            // but were NOT remapped in that partition's scraper run
+            bool consistent = true;
+            string[] archDirs = new string[] { "common", "x86", "x64", "arm64" };
+
+            foreach (string archDir in archDirs)
+            {
+                string dir = Path.Combine(this.GeneratedDir, archDir);
+                if (!Directory.Exists(dir)) continue;
+
+                foreach (string csFile in Directory.GetFiles(dir, "*.cs"))
+                {
+                    string partitionName = Path.GetFileNameWithoutExtension(csFile);
+                    this.perPartitionRemaps.TryGetValue(partitionName, out var partRemaps);
+                    partRemaps ??= new Dictionary<string, string>();
+
+                    string content = File.ReadAllText(csFile);
+
+                    foreach (var kv in this.discoveredRemaps)
+                    {
+                        string tagName = kv.Key;
+                        string typedefName = kv.Value;
+
+                        // Skip if this partition already discovered this remap
+                        if (partRemaps.ContainsKey(tagName)) continue;
+
+                        // Skip if the tag name is also a configured --remap (global)
+                        // These are already handled by scraper.settings.rsp
+                        // Check by looking for the tag in the generated output
+                        // We look for the tag as a word boundary (type usage)
+                        if (content.Contains(tagName) && Regex.IsMatch(content, $@"\b{Regex.Escape(tagName)}\b"))
+                        {
+                            // The tag name appears in this partition's output but was NOT
+                            // remapped by this partition. Check if it's actually used as
+                            // a type (not just in a NativeTypeName attribute string)
+                            var matches = Regex.Matches(content, $@"(?<!\[NativeTypeName\(""[^""]*)\b{Regex.Escape(tagName)}\b(?![^""]*""\)])");
+                            if (matches.Count > 0)
+                            {
+                                // Find which partition DID discover this remap and what header has the typedef
+                                string definingPartition = null;
+                                foreach (var pp in this.perPartitionRemaps)
+                                {
+                                    if (pp.Value.ContainsKey(tagName))
+                                    {
+                                        definingPartition = pp.Key;
+                                        break;
+                                    }
+                                }
+
+                                this.discoveredRemapHeaders.TryGetValue(tagName, out string headerFile);
+                                string headerHint = headerFile != null
+                                    ? $" Add '#include \"{Path.GetFileName(headerFile)}\"' to partition '{partitionName}' main.cpp."
+                                    : "";
+
+                                this.Log.LogError(
+                                    $"Inconsistent remap: '{tagName}' → '{typedefName}' was discovered in partition " +
+                                    $"'{definingPartition}' but appears un-remapped in partition '{partitionName}' " +
+                                    $"(arch: {archDir}).{headerHint}");
+                                consistent = false;
+                            }
+                        }
+                    }
+                }
+            }
+
+            if (!consistent)
+            {
+                this.Log.LogError(
+                    "Cross-partition remap inconsistencies detected. See errors above. " +
+                    "Fix by adding the missing #include directives to the affected partition's main.cpp " +
+                    "so the typedef is visible during scraping.");
+            }
+
+            return consistent;
+        }
+
     }
 }
diff --git a/sources/Win32MetadataScraper/Program.cs b/sources/Win32MetadataScraper/Program.cs
@@ -5,7 +5,7 @@
 // with the configured --remap entries, then runs PInvokeGenerator.GenerateBindings
 // with the full merged remap set — all in a single parse pass.
 //
-// Usage: dotnet Win32MetadataScraper.dll <nativeLibDir> <remapsOutputPath> [--remap-exclusions <file>] @rsp1 @rsp2 ...
+// Usage: dotnet Win32MetadataScraper.dll <remapsOutputPath> @rsp1 @rsp2 ...
 
 using System;
 using System.Collections.Generic;
@@ -24,20 +24,14 @@ class Program
 {
     static int Main(string[] args)
     {
-        if (args.Length < 3)
+        if (args.Length < 2)
         {
-            Console.Error.WriteLine("Usage: dotnet Win32MetadataScraper.dll <nativeLibDir> <remapsOutputPath> @rsp1 @rsp2 ...");
+            Console.Error.WriteLine("Usage: dotnet Win32MetadataScraper.dll <remapsOutputPath> @rsp1 @rsp2 ...");
             return 1;
         }
 
-        // Pre-load native libraries from the specified directory (the ClangSharp tool store)
-        string nativeLibDir = args[0];
-        string remapsOutputPath = args[1];
-
-        NativeLibrary.Load(Path.Combine(nativeLibDir, "libclang.dll"));
-        NativeLibrary.Load(Path.Combine(nativeLibDir, "libClangSharp.dll"));
-
-        string[] rspArgs = args.Skip(2).ToArray();
+        string remapsOutputPath = args[0];
+        string[] rspArgs = args.Skip(1).ToArray();
 
         try
         {
@@ -185,9 +179,15 @@ static int Main(string[] args)
 
                 using var writer = new StreamWriter(remapsOutputPath);
 
-                // Tag remaps
+                // Tag remaps (with header file info)
                 foreach (var kv in allAutoRemaps.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase))
-                    writer.WriteLine($"{kv.Key}={kv.Value}");
+                {
+                    discovery.TagToHeaderFile.TryGetValue(kv.Key, out string srcHeader);
+                    if (srcHeader != null)
+                        writer.WriteLine($"{kv.Key}={kv.Value}|HEADER:{srcHeader}");
+                    else
+                        writer.WriteLine($"{kv.Key}={kv.Value}");
+                }
 
                 // Function pointer excludes (prefixed for downstream parsing)
                 foreach (var excl in fnPtrResult.FnPtrExcludes.OrderBy(x => x, StringComparer.OrdinalIgnoreCase))
diff --git a/sources/Win32MetadataScraper/RemapDiscovery.cs b/sources/Win32MetadataScraper/RemapDiscovery.cs
diff --git a/sources/Win32MetadataScraper/Win32MetadataScraper.csproj b/sources/Win32MetadataScraper/Win32MetadataScraper.csproj
diff --git a/tests/Win32MetadataScraperTests/HeaderSnippetParser.cs b/tests/Win32MetadataScraperTests/HeaderSnippetParser.cs
diff --git a/tests/Win32MetadataScraperTests/RemapDiscoveryTests.cs b/tests/Win32MetadataScraperTests/RemapDiscoveryTests.cs
diff --git a/tests/Win32MetadataScraperTests/Win32MetadataScraperTests.csproj b/tests/Win32MetadataScraperTests/Win32MetadataScraperTests.csproj

Original file line number	Diff line number	Diff line change
`@@ -19,3 +19,4 @@ extern "C" {`
`19`	`19`	`}`
`20`	`20`	`#include <rtmv2.h>`
`21`	`21`	`#include <raserror.h>`
	`22`	`+#include <inaddr.h>`
Original file line number	Diff line number	Diff line change
`@@ -31,3 +31,4 @@ typedef struct _OLD_LARGE_INTEGER {`
`31`	`31`
`32`	`32`	`#include <wintrust.h>`
`33`	`33`	`#include <softpub.h>`
	`34`	`+#include <mssip.h>`
Original file line number	Diff line number	Diff line change
`@@ -8,3 +8,4 @@`
`8`	`8`
`9`	`9`	`#include <webservices.h>`
`10`	`10`	`#include <icontentprefetchertasktrigger.h>`
	`11`	`+#include <schannel.h>`