Users/jcfiorenzano23/go 117 experiment (#1359)

* Created experimental go detector

* Created experiment configuration

* Added unit tests

* Registered the interfaces

* Resolved issue resolving ILogger

* Updated detector version and using the env var to determine if the graph should run

Author: jcfiorenzano

src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs

@@ -139,11 +139,28 @@ private async Task<IndividualDetectorScanResult> ProcessAsync(
         };
     }
 
+    /// <summary>
+    /// Auxliary method executed before the actual scanning of a given file takes place.
+    /// This method can be used to modify or create new ProcessRequests that later will
+    /// be used by the scanner to extract the components.
+    /// </summary>
+    /// <param name="processRequests">Process requests that triggered a given scanner.</param>
+    /// <param name="detectorArgs">Arguments used by the detector.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>Returns the process requests that will be used by the scanner.</returns>
     protected virtual Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(IObservable<ProcessRequest> processRequests, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
     {
         return Task.FromResult(processRequests);
     }
 
+    /// <summary>
+    /// This method called for each individual process request.
+    /// It is responsible for the actual scanning of the components.
+    /// </summary>
+    /// <param name="processRequest">Process request that contains information to execute the component scan.</param>
+    /// <param name="detectorArgs">Arguments for the detector.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>A task representing the asynchronous operation.</returns>
     protected abstract Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default);
 
     protected virtual Task OnDetectionFinishedAsync()

src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs

@@ -0,0 +1,201 @@
+namespace Microsoft.ComponentDetection.Detectors.Go;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.ComponentDetection.Common;
+using Microsoft.ComponentDetection.Common.Telemetry.Records;
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Contracts.Internal;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.Extensions.Logging;
+
+public class Go117ComponentDetector : FileComponentDetector, IExperimentalDetector
+{
+    private readonly HashSet<string> projectRoots = [];
+
+    private readonly ICommandLineInvocationService commandLineInvocationService;
+    private readonly IGoParserFactory goParserFactory;
+    private readonly IEnvironmentVariableService envVarService;
+
+    public Go117ComponentDetector(
+        IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
+        IObservableDirectoryWalkerFactory walkerFactory,
+        ICommandLineInvocationService commandLineInvocationService,
+        IEnvironmentVariableService envVarService,
+        ILogger<GoComponentDetector> logger,
+        IFileUtilityService fileUtilityService,
+        IGoParserFactory goParserFactory)
+    {
+        this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
+        this.Scanner = walkerFactory;
+        this.commandLineInvocationService = commandLineInvocationService;
+        this.Logger = logger;
+        this.goParserFactory = goParserFactory;
+        this.envVarService = envVarService;
+    }
+
+    public override string Id => "Go117";
+
+    public override IEnumerable<string> Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.GoMod)];
+
+    public override IList<string> SearchPatterns { get; } = ["go.mod", "go.sum"];
+
+    public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = [ComponentType.Go];
+
+    public override int Version => 1;
+
+    protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
+        IObservable<ProcessRequest> processRequests,
+        IDictionary<string, string> detectorArgs,
+        CancellationToken cancellationToken = default)
+    {
+        var goModProcessRequests = processRequests.Where(processRequest =>
+        {
+            if (Path.GetFileName(processRequest.ComponentStream.Location) != "go.sum")
+            {
+                return true;
+            }
+
+            var goModFile = this.FindAdjacentGoModComponentStreams(processRequest).FirstOrDefault();
+
+            try
+            {
+                if (goModFile == null)
+                {
+                    this.Logger.LogDebug(
+                        "go.sum file found without an adjacent go.mod file. Location: {Location}",
+                        processRequest.ComponentStream.Location);
+
+                    return true;
+                }
+
+                return GoDetectorUtils.ShouldRemoveGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger);
+            }
+            finally
+            {
+                goModFile?.Stream.Dispose();
+            }
+        });
+
+        return Task.FromResult(goModProcessRequests);
+    }
+
+    protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
+    {
+        var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder;
+        var file = processRequest.ComponentStream;
+
+        var projectRootDirectory = Directory.GetParent(file.Location);
+        if (this.projectRoots.Any(path => projectRootDirectory.FullName.StartsWith(path)))
+        {
+            return;
+        }
+
+        var record = new GoGraphTelemetryRecord();
+        var fileExtension = Path.GetExtension(file.Location).ToUpperInvariant();
+        switch (fileExtension)
+        {
+            case ".MOD":
+            {
+                this.Logger.LogDebug("Found Go.mod: {Location}", file.Location);
+                await this.goParserFactory.CreateParser(GoParserType.GoMod, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
+
+                if (await this.ShouldRunGoGraphAsync())
+                {
+                    await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync(
+                        this.commandLineInvocationService,
+                        this.Logger,
+                        singleFileComponentRecorder,
+                        projectRootDirectory.FullName,
+                        record,
+                        cancellationToken);
+                }
+
+                break;
+            }
+
+            case ".SUM":
+            {
+                this.Logger.LogDebug("Found Go.sum: {Location}", file.Location);
+                await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
+                break;
+            }
+
+            default:
+            {
+                throw new InvalidOperationException("Unexpected file type detected in go detector");
+            }
+        }
+    }
+
+    private bool IsGoCliManuallyDisabled()
+    {
+        return this.envVarService.IsEnvironmentVariableValueTrue("DisableGoCliScan");
+    }
+
+    private async Task<bool> ShouldRunGoGraphAsync()
+    {
+        if (this.IsGoCliManuallyDisabled())
+        {
+            return false;
+        }
+
+        var goVersion = await this.GetGoVersionAsync();
+        if (goVersion == null)
+        {
+            return false;
+        }
+
+        return goVersion >= new Version(1, 11);
+    }
+
+    private async Task<Version> GetGoVersionAsync()
+    {
+        var processExecution = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, null, cancellationToken: default, new List<string> { "version" }.ToArray());
+        if (processExecution.ExitCode != 0)
+        {
+            return null;
+        }
+
+        // Define the regular expression pattern to match the version number
+        var versionPattern = @"go version go(\d+\.\d+\.\d+)";
+        var match = Regex.Match(processExecution.StdOut, versionPattern);
+
+        if (match.Success)
+        {
+            // Extract the version number from the match
+            var versionStr = match.Groups[1].Value;
+            return new Version(versionStr);
+        }
+
+        return null;
+    }
+
+    private IEnumerable<ComponentStream> FindAdjacentGoModComponentStreams(ProcessRequest processRequest) =>
+        this.ComponentStreamEnumerableFactory.GetComponentStreams(
+                new FileInfo(processRequest.ComponentStream.Location).Directory,
+                ["go.mod"],
+                (_, _) => false,
+                false)
+            .Select(x =>
+            {
+                // The stream will be disposed at the end of this method, so we need to copy it to a new stream.
+                var memoryStream = new MemoryStream();
+
+                x.Stream.CopyTo(memoryStream);
+                memoryStream.Position = 0;
+
+                return new ComponentStream
+                {
+                    Stream = memoryStream,
+                    Location = x.Location,
+                    Pattern = x.Pattern,
+                };
+            });
+}