Feature/sn 959 create performance tests for analyzers

.github/workflows/run-code-analyzers-benchmarks.yml

@@ -0,0 +1,128 @@
+# ==============================================================================
+# Code Analyzers Performance Comparison
+#
+# Triggered on pull requests that modify Saritasa.Tools.CodeAnalyzers project.
+#
+# How it works:
+#   1. Checks out the PR branch and runs benchmarks against an external test project.
+#   2. Switches only the Saritasa.Tools.CodeAnalyzers source to the base branch (master),
+#      keeping the benchmark code and test project unchanged.
+#   3. Runs benchmarks again to get the baseline results.
+#   4. Compares the two runs and fails if mean time or allocations regress
+#      beyond the configured thresholds (THRESHOLD_MEAN / THRESHOLD_ALLOCATION).
+#   5. Posts the comparison report as a sticky PR comment.
+#
+# More info about tool used to analyze and compare at https://github.com/TechNobre/PowerUtils.BenchmarkDotnet.Reporter.
+# ==============================================================================
+name: Code Analyzers Performance Comparison
+
+on:
+  pull_request:
+    branches:
+      - master
+
+    paths:
+      - 'src/Saritasa.Tools.CodeAnalyzers/**'
+
+   # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  PATH_BENCHMARKS_PROJECT: 'test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Saritasa.Tools.CodeAnalyzers.Benchmarks.csproj'
+  DIR_BASELINE_REPORTS: './artifacts-baseline'
+  DIR_TARGET_REPORTS: './artifacts-target'
+  PATH_REPORT_RESULT: './BenchmarkReporter/benchmark-comparison-report.md'
+  TEST_PROJECT_FOLDER: './testProject'
+  PATH_TO_TEST_SOLUTION: 'src/Saritasa.NetForge.slnx'
+  THRESHOLD_MEAN: 50%
+  THRESHOLD_ALLOCATION: 50%
+
+jobs:
+  compare-code-analyzers-performance:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Ensure a clean test project directory before checkout.
+      - name: Clean up target directory
+        run: rm -rf ${{ env.TEST_PROJECT_FOLDER }}
+
+      # Checkout the PR branch so we can benchmark the proposed changes.
+      - name: Checkout PR Branch (Current Changes)
+        uses: actions/checkout@v4
+
+      # Checkout the external test project used as input for the analyzers.
+      - name: Checkout External Repository
+        id: external_checkout
+        uses: actions/checkout@v4
+        with:
+          repository: 'saritasa-nest/saritasa-forge-admin'
+          path: ${{ env.TEST_PROJECT_FOLDER }}
+          persist-credentials: false
+          ssh-strict: false
+          set-safe-directory: false
+          fetch-depth: 1
+
+      # Pin the test project to a known commit for reproducible benchmark input.
+      - name: Reset to specific commit
+        uses: actions/checkout@v4
+        with:
+            repository: 'saritasa-nest/saritasa-forge-admin'
+            ref: dc2d1ed5d58be20e709de5ba884beeaba0c6a733
+            path: ${{ env.TEST_PROJECT_FOLDER }}
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '10.0.x'
+
+      # Benchmark the PR branch. sudo -E preserves the environment so this allows
+      # BenchmarkDotNet to raise the process priority for more stable measurements.
+      - name: Run PR Benchmarks
+        run: |
+          sudo -E dotnet run --no-launch-profile -c Release \
+            --project ${{ env.PATH_BENCHMARKS_PROJECT }} -- \
+            --artifacts ${{ env.DIR_TARGET_REPORTS }} \
+            --testProjectPath ${{ env.TEST_PROJECT_FOLDER }}/${{ env.PATH_TO_TEST_SOLUTION }}
+
+      # Swap the Saritasa.Tools.CodeAnalyzers code to the base branch while keeping the test project intact.
+      # The directory is removed first to ensure files added in the PR (not present in base) are deleted.
+      # Without this, `git checkout FETCH_HEAD -- <path>` only updates existing files and leaves new ones behind.
+      - name: Switch Saritasa.Tools.CodeAnalyzers to Base Branch
+        run: |
+          git fetch origin ${{ github.base_ref }} --depth=1
+          sudo rm -rf src/Saritasa.Tools.CodeAnalyzers/
+          git checkout FETCH_HEAD -- src/Saritasa.Tools.CodeAnalyzers/
+          git status
+
+      # Benchmark the base branch with the same test project for a fair comparison.
+      - name: Run Base Benchmarks
+        run: |
+          sudo -E dotnet run --no-launch-profile -c Release \
+            --project ${{ env.PATH_BENCHMARKS_PROJECT }} -- \
+            --artifacts ${{ env.DIR_BASELINE_REPORTS }} \
+            --testProjectPath ${{ env.TEST_PROJECT_FOLDER }}/${{ env.PATH_TO_TEST_SOLUTION }}
+
+      - name: Install lib for benchmarks comparison
+        run: sudo -E dotnet tool install --global PowerUtils.BenchmarkDotnet.Reporter
+
+      - name: Run benchmarks compare
+        run: |
+          pbreporter compare \
+            -b ${{ env.DIR_BASELINE_REPORTS }}/results \
+            -t ${{ env.DIR_TARGET_REPORTS }}/results \
+            -f markdown \
+            -tm ${{ env.THRESHOLD_MEAN }} \
+            -ta ${{ env.THRESHOLD_ALLOCATION }} \
+            -ft -fw
+
+      - name: Publish benchmark report in Summary
+        run: cat ${{ env.PATH_REPORT_RESULT }} > $GITHUB_STEP_SUMMARY
+
+      - name: Add compare benchmark report in PR Comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        if: github.event_name == 'pull_request'
+        with:
+          header: compare-benchmark-report
+          hide_and_recreate: true
+          hide_classify: "OUTDATED"
+          path: ${{ env.PATH_REPORT_RESULT }}

Saritasa.Tools.sln

@@ -45,6 +45,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Saritasa.Tools.CodeAnalyzer
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Saritasa.Tools.CodeAnalyzers.Tests", "test\Saritasa.Tools.CodeAnalyzers.Tests\Saritasa.Tools.CodeAnalyzers.Tests.csproj", "{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Saritasa.Tools.CodeAnalyzers.Benchmarks", "test\Saritasa.Tools.CodeAnalyzers.Benchmarks\Saritasa.Tools.CodeAnalyzers.Benchmarks.csproj", "{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -111,6 +113,10 @@ Global
 		{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -131,6 +137,7 @@ Global
 		{31FDE475-97E3-49C0-85BA-FD782061514B} = {635DB5B1-86B3-4F5B-97A6-04C71D672296}
 		{29B003C4-F226-47AC-B591-3C563E6667AB} = {45D13EC5-88F3-403A-8148-0812C2796062}
 		{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2} = {635DB5B1-86B3-4F5B-97A6-04C71D672296}
+		{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F} = {635DB5B1-86B3-4F5B-97A6-04C71D672296}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F298C89B-4162-49D5-B517-5306BD58DB59}

test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkExporter.cs

@@ -0,0 +1,34 @@
+using BenchmarkDotNet.Exporters.Json;
+using BenchmarkDotNet.Reports;
+
+namespace Saritasa.Tools.CodeAnalyzers.Benchmarks;
+
+/// <summary>
+/// A custom JSON exporter that replaces the benchmark method name with the value of its first parameter
+/// for a more convenient display of code analyzer test results.
+/// </summary>
+public class CodeAnalyzersBenchmarkExporter : JsonExporterBase
+{
+    /// <summary>
+    /// Constructor.
+    /// </summary>
+    public CodeAnalyzersBenchmarkExporter() : base(indentJson: true, excludeMeasurements: true)
+    {
+    }
+
+    /// <inheritdoc/>
+    protected override IReadOnlyDictionary<string, object> GetDataToSerialize(BenchmarkReport report)
+    {
+        var dict = base.GetDataToSerialize(report);
+
+        var firstParam = report.BenchmarkCase.Parameters.Items.FirstOrDefault();
+        var methodName = firstParam?.Value?.ToString() ?? report.BenchmarkCase.Descriptor.WorkloadMethod.Name;
+
+        var copy = dict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
+
+        copy["Method"] = methodName;
+        copy["MethodTitle"] = methodName;
+
+        return copy;
+    }
+}

test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkSettings.cs

@@ -0,0 +1,12 @@
+namespace Saritasa.Tools.CodeAnalyzers.Benchmarks;
+
+/// <summary>
+/// Benchmark run settings.
+/// </summary>
+internal static class CodeAnalyzersBenchmarkSettings
+{
+    /// <summary>
+    /// Path to the test project solution file passed via --testProjectPath argument.
+    /// </summary>
+    public static string TestProjectPath { get; set; } = string.Empty;
+}

test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarks.cs

@@ -0,0 +1,80 @@
+using BenchmarkDotNet.Attributes;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.MSBuild;
+using Saritasa.Tools.CodeAnalyzers.Analyzers;
+
+namespace Saritasa.Tools.CodeAnalyzers.Benchmarks;
+
+/// <summary>
+/// Wraps a DiagnosticAnalyzer to provide a short display name for BenchmarkDotNet.
+/// </summary>
+public class AnalyzerParam(DiagnosticAnalyzer analyzer)
+{
+    /// <summary>
+    /// Analyzer.
+    /// </summary>
+    public DiagnosticAnalyzer Analyzer { get; } = analyzer;
+
+    /// <inheritdoc/>
+    public override string ToString() => Analyzer.GetType().Name;
+}
+
+/// <summary>
+/// Benchmarks for Roslyn diagnostic analyzers.
+/// Analyzers are discovered dynamically via reflection from Saritasa.Tools.CodeAnalyzers assembly,
+/// so no changes to this file are needed when a new analyzer is added.
+/// </summary>
+public class CodeAnalyzersBenchmarks
+{
+    private static readonly List<Compilation> compilations = new();
+
+    /// <summary>
+    /// Analyzer source.
+    /// </summary>
+    public static IEnumerable<AnalyzerParam> AnalyzerSource { get; } =
+        typeof(LineLengthAnalyzer).Assembly
+            .GetTypes()
+            .Where(t => !t.IsAbstract && typeof(DiagnosticAnalyzer).IsAssignableFrom(t))
+            .Select(t => new AnalyzerParam((DiagnosticAnalyzer)Activator.CreateInstance(t)!))
+            .ToList();
+
+    static CodeAnalyzersBenchmarks()
+    {
+        using var workspace = MSBuildWorkspace.Create();
+
+        workspace.RegisterWorkspaceFailedHandler(args =>
+        {
+            Console.WriteLine($"[MSBuild] {args.Diagnostic.Message}");
+        });
+
+        var solution = workspace.OpenSolutionAsync(CodeAnalyzersBenchmarkSettings.TestProjectPath).GetAwaiter().GetResult();
+
+        foreach (var project in solution.Projects)
+        {
+            var compilation = project.GetCompilationAsync().GetAwaiter().GetResult();
+            if (compilation == null)
+            {
+                continue;
+            }
+            compilations.Add(compilation);
+        }
+    }
+
+    /// <summary>
+    /// Runs a single analyzer against all compiled projects.
+    /// Generates one benchmark case per analyzer found in AnalyzerSource.
+    /// </summary>
+    [Benchmark]
+    [ArgumentsSource(nameof(AnalyzerSource))]
+    public async Task RunAnalyzer(AnalyzerParam param)
+    {
+        foreach (var compilation in compilations)
+        {
+            // A new instance must be created each iteration: CompilationWithAnalyzers caches
+            // results internally, so reusing it would measure cache retrieval, not actual analysis.
+            var compilationWithAnalyzers = compilation.WithAnalyzers([param.Analyzer]);
+            _ = await compilationWithAnalyzers.GetAnalyzerDiagnosticsAsync();
+        }
+    }
+}

test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Program.cs

@@ -0,0 +1,65 @@
+using System.CommandLine;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using Microsoft.Build.Locator;
+
+namespace Saritasa.Tools.CodeAnalyzers.Benchmarks;
+
+/// <summary>
+/// Program class.
+/// </summary>
+internal class Program
+{
+    /// <summary>
+    /// Entry point.
+    /// </summary>
+    public static async Task<int> Main(string[] args)
+    {
+        var testProjectPathOption = new Option<string>("--testProjectPath")
+        {
+            Description = "Path to the test project for code analyzer benchmarks.",
+            Required = true
+        };
+
+        var rootCommand = new RootCommand();
+        rootCommand.Options.Add(testProjectPathOption);
+        rootCommand.TreatUnmatchedTokensAsErrors = false;
+
+        var parseResult = rootCommand.Parse(args);
+        if (parseResult.Errors.Count == 0 && parseResult.GetValue(testProjectPathOption) is string testProjectPath)
+        {
+            CodeAnalyzersBenchmarkSettings.TestProjectPath = testProjectPath;
+
+            MSBuildLocator.RegisterDefaults();
+
+            var config = ManualConfig.CreateEmpty()
+                // InProcess is required because MSBuildLocator.RegisterDefaults() registers an
+                // assembly resolver in the current AppDomain. The default out-of-process toolchain
+                // spawns a child process with a BenchmarkDotNet-generated Main that never calls
+                // MSBuildLocator, so MSBuild assemblies cannot be resolved.
+                .AddJob(Job.Default
+                    .WithToolchain(InProcessEmitToolchain.Instance))
+                    .WithOption(ConfigOptions.StopOnFirstError, true)
+                .AddLogger(ConsoleLogger.Default)
+                    .WithOption(ConfigOptions.DisableLogFile, true)
+                .AddDiagnoser(MemoryDiagnoser.Default)
+                .AddExporter(new CodeAnalyzersBenchmarkExporter())
+                .AddColumnProvider(DefaultColumnProviders.Instance);
+
+            BenchmarkRunner.Run<CodeAnalyzersBenchmarks>(config, parseResult.UnmatchedTokens.ToArray());
+            return 0;
+        }
+
+        foreach (var parseError in parseResult.Errors)
+        {
+            await Console.Error.WriteLineAsync(parseError.Message);
+        }
+
+        return 1;
+    }
+}

test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Properties/launchSettings.json

@@ -0,0 +1,9 @@
+{
+  "profiles": {
+    "Saritasa.Tools.CodeAnalyzers.Benchmarks": {
+      "workingDirectory": "$(ProjectDir)",
+      "commandName": "Project",
+      "commandLineArgs": "--testProjectPath \"$(SolutionDir)Saritasa.Tools.sln\" --artifacts artifacts"
+    }
+  }
+}