Merge pull request #15518 from tamasvajk/feature/telemetry-dependency-manager

C#: Extract dependency restore telemetry data

Author: tamasvajk

csharp/downgrades/c9ee11bd1ee96e925a35cedff000be924634447f/old.dbscheme

@@ -0,0 +1,3 @@
+description: Remove `compilation_info`.
+compatibility: backwards
+compilation_info.rel: delete

csharp/downgrades/c9ee11bd1ee96e925a35cedff000be924634447f/semmlecode.csharp.dbscheme

@@ -133,6 +133,17 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts");
             logger.LogInfo($"{dotnetFrameworkVersionVariantCount,align} restored .NET framework variants");
             logger.LogInfo($"Build analysis completed in {DateTime.Now - startTime}");
+
+            CompilationInfos.AddRange([
+                ("Source files on filesystem", nonGeneratedSources.Count.ToString()),
+                ("Source files generated", generatedSources.Count.ToString()),
+                ("Solution files on filesystem", allSolutions.Count.ToString()),
+                ("Project files on filesystem", allProjects.Count.ToString()),
+                ("Resolved references", usedReferences.Keys.Count.ToString()),
+                ("Unresolved references", unresolvedReferences.Count.ToString()),
+                ("Resolved assembly conflicts", conflictedReferences.ToString()),
+                ("Restored .NET framework variants", dotnetFrameworkVersionVariantCount.ToString()),
+            ]);
         }
 
         private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
@@ -151,7 +162,13 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<
             try
             {
                 var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);
-                nuget.InstallPackages();
+                var count = nuget.InstallPackages();
+
+                if (nuget.PackageCount > 0)
+                {
+                    CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
+                    CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
+                }
 
                 var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
                 var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
@@ -629,6 +646,11 @@ private void ResolveConflicts(IEnumerable<string> frameworkPaths)
         /// </summary>
         public IEnumerable<string> UnresolvedReferences => unresolvedReferences.Select(r => r.Key);
 
+        /// <summary>
+        /// List of `(key, value)` tuples, that are stored in the DB for telemetry purposes.
+        /// </summary>
+        public List<(string, string)> CompilationInfos { get; } = new List<(string, string)>();
+
         /// <summary>
         /// Record that a particular reference couldn't be resolved.
         /// Note that this records at most one project file per missing reference.
@@ -699,15 +721,22 @@ private void AnalyseProject(FileInfo project)
         /// <param name="solutions">A list of paths to solution files.</param>
         private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
         {
+            var successCount = 0;
             var assetFiles = new List<string>();
             var projects = solutions.SelectMany(solution =>
                 {
                     logger.LogInfo($"Restoring solution {solution}...");
                     var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
                     assetFiles.AddRange(res.AssetsFilePaths);
                     return res.RestoredProjects;
-                });
+                }).ToList();
             assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
+            CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
             return projects;
         }
 
@@ -719,14 +748,24 @@ private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out
         /// <param name="projects">A list of paths to project files.</param>
         private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
         {
+            var successCount = 0;
             var assetFiles = new List<string>();
+            var sync = new object();
             Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
             {
                 logger.LogInfo($"Restoring project {project}...");
                 var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
-                assetFiles.AddRange(res.AssetsFilePaths);
+                lock (sync)
+                {
+                    if (res.Success)
+                    {
+                        successCount++;
+                    }
+                    assetFiles.AddRange(res.AssetsFilePaths);
+                }
             });
             assets = assetFiles;
+            CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
         }
 
         private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
@@ -767,6 +806,11 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
                 logger.LogInfo($"Using nuget.config file {nugetConfig}.");
             }
 
+            CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
+
+            var successCount = 0;
+            var sync = new object();
+
             Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
             {
                 logger.LogInfo($"Restoring package {package}...");
@@ -797,10 +841,18 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
                     if (!res.Success)
                     {
                         logger.LogInfo($"Failed to restore nuget package {package}");
+                        return;
                     }
                 }
+
+                lock (sync)
+                {
+                    successCount++;
+                }
             });
 
+            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
+
             dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
         }
 

csharp/downgrades/c9ee11bd1ee96e925a35cedff000be924634447f/upgrade.properties

@@ -22,6 +22,8 @@ internal class NugetPackages
         /// </summary>
         private readonly FileInfo[] packageFiles;
 
+        public int PackageCount => packageFiles.Length;
+
         /// <summary>
         /// The computed packages directory.
         /// This will be in the Temp location
@@ -105,7 +107,7 @@ private string DownloadNugetExe(string sourceDir)
         /// Restore all files in a specified package.
         /// </summary>
         /// <param name="package">The package file.</param>
-        private void RestoreNugetPackage(string package)
+        private bool TryRestoreNugetPackage(string package)
         {
             logger.LogInfo($"Restoring file {package}...");
 
@@ -141,22 +143,21 @@ private void RestoreNugetPackage(string package)
             if (exitCode != 0)
             {
                 logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
+                return false;
             }
             else
             {
                 logger.LogInfo($"Restored file {package}");
+                return true;
             }
         }
 
         /// <summary>
         /// Download the packages to the temp folder.
         /// </summary>
-        public void InstallPackages()
+        public int InstallPackages()
         {
-            foreach (var package in packageFiles)
-            {
-                RestoreNugetPackage(package.FullName);
-            }
+            return packageFiles.Count(package => TryRestoreNugetPackage(package.FullName));
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -5,14 +5,14 @@
 using System.Linq;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
+using Semmle.Extraction.CSharp.DependencyFetching;
 using Semmle.Util;
 using Semmle.Util.Logging;
 
 namespace Semmle.Extraction.CSharp.Standalone
 {
     public static class Extractor
     {
-
         private static IEnumerable<Action> GetResolvedReferencesStandalone(IEnumerable<string> referencePaths, BlockingCollection<MetadataReference> references)
         {
             return referencePaths.Select<string, Action>(path => () =>
@@ -24,8 +24,7 @@ private static IEnumerable<Action> GetResolvedReferencesStandalone(IEnumerable<s
 
         private static void AnalyseStandalone(
             StandaloneAnalyser analyser,
-            IEnumerable<string> sources,
-            IEnumerable<string> referencePaths,
+            ExtractionInput extractionInput,
             CommonOptions options,
             IProgressMonitor progressMonitor,
             Stopwatch stopwatch)
@@ -35,12 +34,12 @@ private static void AnalyseStandalone(
             try
             {
                 CSharp.Extractor.Analyse(stopwatch, analyser, options,
-                    references => GetResolvedReferencesStandalone(referencePaths, references),
-                    (analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
+                    references => GetResolvedReferencesStandalone(extractionInput.References, references),
+                    (analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(extractionInput.Sources, analyser, null, null, syntaxTrees),
                     (syntaxTrees, references) => CSharpCompilation.Create(
                         output.Name, syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true)
                         ),
-                    (compilation, options) => analyser.Initialize(output.FullName, compilation, options),
+                    (compilation, options) => analyser.Initialize(output.FullName, extractionInput.CompilationInfos, compilation, options),
                     _ => { },
                     () =>
                     {
@@ -73,8 +72,7 @@ private static void AnalyseStandalone(
         }
 
         private static void ExtractStandalone(
-            IEnumerable<string> sources,
-            IEnumerable<string> referencePaths,
+            ExtractionInput extractionInput,
             IProgressMonitor pm,
             ILogger logger,
             CommonOptions options)
@@ -88,7 +86,7 @@ private static void ExtractStandalone(
             using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
             try
             {
-                AnalyseStandalone(analyser, sources, referencePaths, options, pm, stopwatch);
+                AnalyseStandalone(analyser, extractionInput, options, pm, stopwatch);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -131,17 +129,18 @@ public void MissingSummary(int missingTypes, int missingNamespaces)
             }
         }
 
+        public record ExtractionInput(IEnumerable<string> Sources, IEnumerable<string> References, IEnumerable<(string, string)> CompilationInfos);
+
         public static ExitCode Run(Options options)
         {
             var stopwatch = new Stopwatch();
             stopwatch.Start();
 
             using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
             logger.Log(Severity.Info, "Extracting C# in buildless mode");
-            using var a = new Analysis(logger, options);
-            var sourceFileCount = a.Extraction.Sources.Count;
+            using var dependencyManager = new DependencyManager(options.SrcDir, options.Dependencies, logger);
 
-            if (sourceFileCount == 0)
+            if (!dependencyManager.AllSourceFiles.Any())
             {
                 logger.Log(Severity.Error, "No source files found");
                 return ExitCode.Errors;
@@ -152,8 +151,7 @@ public static ExitCode Run(Options options)
             logger.Log(Severity.Info, "");
             logger.Log(Severity.Info, "Extracting...");
             ExtractStandalone(
-                a.Extraction.Sources,
-                a.References,
+                new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos),
                 new ExtractionProgress(logger),
                 fileLogger,
                 options);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs

@@ -5,48 +5,6 @@
 
 namespace Semmle.Extraction.CSharp.Standalone
 {
-    /// <summary>
-    ///     One independent run of the extractor.
-    /// </summary>
-    internal class Extraction
-    {
-        public Extraction(string directory)
-        {
-            Directory = directory;
-        }
-
-        public string Directory { get; }
-        public List<string> Sources { get; } = new List<string>();
-    };
-
-    /// <summary>
-    ///     Searches for source/references and creates separate extractions.
-    /// </summary>
-    internal sealed class Analysis : IDisposable
-    {
-        public Analysis(ILogger logger, Options options)
-        {
-            dependencyManager = new DependencyManager(options.SrcDir, options.Dependencies, logger);
-            References = dependencyManager.ReferenceFiles;
-            Extraction = new Extraction(options.SrcDir);
-            Extraction.Sources.AddRange(dependencyManager.AllSourceFiles);
-        }
-
-        public IEnumerable<string> References { get; }
-
-        /// <summary>
-        /// The extraction configuration.
-        /// </summary>
-        public Extraction Extraction { get; }
-
-        private readonly DependencyManager dependencyManager;
-
-        public void Dispose()
-        {
-            dependencyManager.Dispose();
-        }
-    };
-
     public class Program
     {
         public static int Main(string[] args)

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -6,6 +6,7 @@
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
+using Semmle.Util;
 using Semmle.Util.Logging;
 using Semmle.Extraction.CSharp.Populators;
 
@@ -240,6 +241,8 @@ private void DoAnalyseCompilation()
                 var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
 
                 compilationEntity = Entities.Compilation.Create(cx);
+
+                extractor.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs

@@ -13,10 +13,10 @@ public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyT
         {
         }
 
-        public void Initialize(string outputPath, CSharpCompilation compilationIn, CommonOptions options)
+        public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
         {
             compilation = compilationIn;
-            extractor = new StandaloneExtractor(outputPath, Logger, PathTransformer, options);
+            extractor = new StandaloneExtractor(outputPath, compilationInfos, Logger, PathTransformer, options);
             this.options = options;
             LogExtractorInfo(Extraction.Extractor.Version);
             SetReferencePaths();

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -71,6 +71,9 @@ internal static void compilation_args(this TextWriter trapFile, Compilation comp
         internal static void compilation_expanded_args(this TextWriter trapFile, Compilation compilation, int index, string arg) =>
             trapFile.WriteTuple("compilation_expanded_args", compilation, index, arg);
 
+        internal static void compilation_info(this TextWriter trapFile, Compilation compilation, string infoKey, string infoValue) =>
+            trapFile.WriteTuple("compilation_info", compilation, infoKey, infoValue);
+
         internal static void compilation_compiling_files(this TextWriter trapFile, Compilation compilation, int index, Extraction.Entities.File file) =>
             trapFile.WriteTuple("compilation_compiling_files", compilation, index, file);
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

@@ -1,6 +1,8 @@
 using System.Collections.Generic;
 using Semmle.Util.Logging;
 
+using CompilationInfo = (string key, string value);
+
 namespace Semmle.Extraction
 {
     /// <summary>
@@ -10,17 +12,19 @@ public abstract class Extractor
     {
         public abstract ExtractorMode Mode { get; }
         public string OutputPath { get; }
+        public IEnumerable<CompilationInfo> CompilationInfos { get; }
 
         /// <summary>
         /// Creates a new extractor instance for one compilation unit.
         /// </summary>
         /// <param name="logger">The object used for logging.</param>
         /// <param name="pathTransformer">The object used for path transformations.</param>
-        protected Extractor(string outputPath, ILogger logger, PathTransformer pathTransformer)
+        protected Extractor(string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
         {
             OutputPath = outputPath;
             Logger = logger;
             PathTransformer = pathTransformer;
+            CompilationInfos = compilationInfos;
         }
 
         // Limit the number of error messages in the log file