diff --git a/.github/workflows/run-code-analyzers-benchmarks.yml b/.github/workflows/run-code-analyzers-benchmarks.yml new file mode 100644 index 00000000..9f27203d --- /dev/null +++ b/.github/workflows/run-code-analyzers-benchmarks.yml @@ -0,0 +1,128 @@ +# ============================================================================== +# Code Analyzers Performance Comparison +# +# Triggered on pull requests that modify Saritasa.Tools.CodeAnalyzers project. +# +# How it works: +# 1. Checks out the PR branch and runs benchmarks against an external test project. +# 2. Switches only the Saritasa.Tools.CodeAnalyzers source to the base branch (master), +# keeping the benchmark code and test project unchanged. +# 3. Runs benchmarks again to get the baseline results. +# 4. Compares the two runs and fails if mean time or allocations regress +# beyond the configured thresholds (THRESHOLD_MEAN / THRESHOLD_ALLOCATION). +# 5. Posts the comparison report as a sticky PR comment. +# +# More info about tool used to analyze and compare at https://github.com/TechNobre/PowerUtils.BenchmarkDotnet.Reporter. +# ============================================================================== +name: Code Analyzers Performance Comparison + +on: + pull_request: + branches: + - master + + paths: + - 'src/Saritasa.Tools.CodeAnalyzers/**' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + PATH_BENCHMARKS_PROJECT: 'test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Saritasa.Tools.CodeAnalyzers.Benchmarks.csproj' + DIR_BASELINE_REPORTS: './artifacts-baseline' + DIR_TARGET_REPORTS: './artifacts-target' + PATH_REPORT_RESULT: './BenchmarkReporter/benchmark-comparison-report.md' + TEST_PROJECT_FOLDER: './testProject' + PATH_TO_TEST_SOLUTION: 'src/Saritasa.NetForge.slnx' + THRESHOLD_MEAN: 50% + THRESHOLD_ALLOCATION: 50% + +jobs: + compare-code-analyzers-performance: + runs-on: ubuntu-latest + + steps: + # Ensure a clean test project directory before checkout. + - name: Clean up target directory + run: rm -rf ${{ env.TEST_PROJECT_FOLDER }} + + # Checkout the PR branch so we can benchmark the proposed changes. + - name: Checkout PR Branch (Current Changes) + uses: actions/checkout@v4 + + # Checkout the external test project used as input for the analyzers. + - name: Checkout External Repository + id: external_checkout + uses: actions/checkout@v4 + with: + repository: 'saritasa-nest/saritasa-forge-admin' + path: ${{ env.TEST_PROJECT_FOLDER }} + persist-credentials: false + ssh-strict: false + set-safe-directory: false + fetch-depth: 1 + + # Pin the test project to a known commit for reproducible benchmark input. + - name: Reset to specific commit + uses: actions/checkout@v4 + with: + repository: 'saritasa-nest/saritasa-forge-admin' + ref: dc2d1ed5d58be20e709de5ba884beeaba0c6a733 + path: ${{ env.TEST_PROJECT_FOLDER }} + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + # Benchmark the PR branch. sudo -E preserves the environment so this allows + # BenchmarkDotNet to raise the process priority for more stable measurements. + - name: Run PR Benchmarks + run: | + sudo -E dotnet run --no-launch-profile -c Release \ + --project ${{ env.PATH_BENCHMARKS_PROJECT }} -- \ + --artifacts ${{ env.DIR_TARGET_REPORTS }} \ + --testProjectPath ${{ env.TEST_PROJECT_FOLDER }}/${{ env.PATH_TO_TEST_SOLUTION }} + + # Swap the Saritasa.Tools.CodeAnalyzers code to the base branch while keeping the test project intact. + # The directory is removed first to ensure files added in the PR (not present in base) are deleted. + # Without this, `git checkout FETCH_HEAD -- ` only updates existing files and leaves new ones behind. + - name: Switch Saritasa.Tools.CodeAnalyzers to Base Branch + run: | + git fetch origin ${{ github.base_ref }} --depth=1 + sudo rm -rf src/Saritasa.Tools.CodeAnalyzers/ + git checkout FETCH_HEAD -- src/Saritasa.Tools.CodeAnalyzers/ + git status + + # Benchmark the base branch with the same test project for a fair comparison. + - name: Run Base Benchmarks + run: | + sudo -E dotnet run --no-launch-profile -c Release \ + --project ${{ env.PATH_BENCHMARKS_PROJECT }} -- \ + --artifacts ${{ env.DIR_BASELINE_REPORTS }} \ + --testProjectPath ${{ env.TEST_PROJECT_FOLDER }}/${{ env.PATH_TO_TEST_SOLUTION }} + + - name: Install lib for benchmarks comparison + run: sudo -E dotnet tool install --global PowerUtils.BenchmarkDotnet.Reporter + + - name: Run benchmarks compare + run: | + pbreporter compare \ + -b ${{ env.DIR_BASELINE_REPORTS }}/results \ + -t ${{ env.DIR_TARGET_REPORTS }}/results \ + -f markdown \ + -tm ${{ env.THRESHOLD_MEAN }} \ + -ta ${{ env.THRESHOLD_ALLOCATION }} \ + -ft -fw + + - name: Publish benchmark report in Summary + run: cat ${{ env.PATH_REPORT_RESULT }} > $GITHUB_STEP_SUMMARY + + - name: Add compare benchmark report in PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event_name == 'pull_request' + with: + header: compare-benchmark-report + hide_and_recreate: true + hide_classify: "OUTDATED" + path: ${{ env.PATH_REPORT_RESULT }} diff --git a/Saritasa.Tools.sln b/Saritasa.Tools.sln index 25af97e1..1604111f 100644 --- a/Saritasa.Tools.sln +++ b/Saritasa.Tools.sln @@ -45,6 +45,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Saritasa.Tools.CodeAnalyzer EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Saritasa.Tools.CodeAnalyzers.Tests", "test\Saritasa.Tools.CodeAnalyzers.Tests\Saritasa.Tools.CodeAnalyzers.Tests.csproj", "{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Saritasa.Tools.CodeAnalyzers.Benchmarks", "test\Saritasa.Tools.CodeAnalyzers.Benchmarks\Saritasa.Tools.CodeAnalyzers.Benchmarks.csproj", "{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -111,6 +113,10 @@ Global {908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}.Debug|Any CPU.Build.0 = Debug|Any CPU {908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}.Release|Any CPU.ActiveCfg = Release|Any CPU {908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}.Release|Any CPU.Build.0 = Release|Any CPU + {03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -131,6 +137,7 @@ Global {31FDE475-97E3-49C0-85BA-FD782061514B} = {635DB5B1-86B3-4F5B-97A6-04C71D672296} {29B003C4-F226-47AC-B591-3C563E6667AB} = {45D13EC5-88F3-403A-8148-0812C2796062} {908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2} = {635DB5B1-86B3-4F5B-97A6-04C71D672296} + {03FB7D6A-2A23-4BB8-A2C3-87A5B791832F} = {635DB5B1-86B3-4F5B-97A6-04C71D672296} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {F298C89B-4162-49D5-B517-5306BD58DB59} diff --git a/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkExporter.cs b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkExporter.cs new file mode 100644 index 00000000..25346a61 --- /dev/null +++ b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkExporter.cs @@ -0,0 +1,34 @@ +using BenchmarkDotNet.Exporters.Json; +using BenchmarkDotNet.Reports; + +namespace Saritasa.Tools.CodeAnalyzers.Benchmarks; + +/// +/// A custom JSON exporter that replaces the benchmark method name with the value of its first parameter +/// for a more convenient display of code analyzer test results. +/// +public class CodeAnalyzersBenchmarkExporter : JsonExporterBase +{ + /// + /// Constructor. + /// + public CodeAnalyzersBenchmarkExporter() : base(indentJson: true, excludeMeasurements: true) + { + } + + /// + protected override IReadOnlyDictionary GetDataToSerialize(BenchmarkReport report) + { + var dict = base.GetDataToSerialize(report); + + var firstParam = report.BenchmarkCase.Parameters.Items.FirstOrDefault(); + var methodName = firstParam?.Value?.ToString() ?? report.BenchmarkCase.Descriptor.WorkloadMethod.Name; + + var copy = dict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + + copy["Method"] = methodName; + copy["MethodTitle"] = methodName; + + return copy; + } +} diff --git a/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkSettings.cs b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkSettings.cs new file mode 100644 index 00000000..3dc7f3df --- /dev/null +++ b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkSettings.cs @@ -0,0 +1,12 @@ +namespace Saritasa.Tools.CodeAnalyzers.Benchmarks; + +/// +/// Benchmark run settings. +/// +internal static class CodeAnalyzersBenchmarkSettings +{ + /// + /// Path to the test project solution file passed via --testProjectPath argument. + /// + public static string TestProjectPath { get; set; } = string.Empty; +} diff --git a/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarks.cs b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarks.cs new file mode 100644 index 00000000..134b350a --- /dev/null +++ b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarks.cs @@ -0,0 +1,80 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.MSBuild; +using Saritasa.Tools.CodeAnalyzers.Analyzers; + +namespace Saritasa.Tools.CodeAnalyzers.Benchmarks; + +/// +/// Wraps a DiagnosticAnalyzer to provide a short display name for BenchmarkDotNet. +/// +public class AnalyzerParam(DiagnosticAnalyzer analyzer) +{ + /// + /// Analyzer. + /// + public DiagnosticAnalyzer Analyzer { get; } = analyzer; + + /// + public override string ToString() => Analyzer.GetType().Name; +} + +/// +/// Benchmarks for Roslyn diagnostic analyzers. +/// Analyzers are discovered dynamically via reflection from Saritasa.Tools.CodeAnalyzers assembly, +/// so no changes to this file are needed when a new analyzer is added. +/// +public class CodeAnalyzersBenchmarks +{ + private static readonly List compilations = new(); + + /// + /// Analyzer source. + /// + public static IEnumerable AnalyzerSource { get; } = + typeof(LineLengthAnalyzer).Assembly + .GetTypes() + .Where(t => !t.IsAbstract && typeof(DiagnosticAnalyzer).IsAssignableFrom(t)) + .Select(t => new AnalyzerParam((DiagnosticAnalyzer)Activator.CreateInstance(t)!)) + .ToList(); + + static CodeAnalyzersBenchmarks() + { + using var workspace = MSBuildWorkspace.Create(); + + workspace.RegisterWorkspaceFailedHandler(args => + { + Console.WriteLine($"[MSBuild] {args.Diagnostic.Message}"); + }); + + var solution = workspace.OpenSolutionAsync(CodeAnalyzersBenchmarkSettings.TestProjectPath).GetAwaiter().GetResult(); + + foreach (var project in solution.Projects) + { + var compilation = project.GetCompilationAsync().GetAwaiter().GetResult(); + if (compilation == null) + { + continue; + } + compilations.Add(compilation); + } + } + + /// + /// Runs a single analyzer against all compiled projects. + /// Generates one benchmark case per analyzer found in AnalyzerSource. + /// + [Benchmark] + [ArgumentsSource(nameof(AnalyzerSource))] + public async Task RunAnalyzer(AnalyzerParam param) + { + foreach (var compilation in compilations) + { + // A new instance must be created each iteration: CompilationWithAnalyzers caches + // results internally, so reusing it would measure cache retrieval, not actual analysis. + var compilationWithAnalyzers = compilation.WithAnalyzers([param.Analyzer]); + _ = await compilationWithAnalyzers.GetAnalyzerDiagnosticsAsync(); + } + } +} diff --git a/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Program.cs b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Program.cs new file mode 100644 index 00000000..5a341361 --- /dev/null +++ b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Program.cs @@ -0,0 +1,65 @@ +using System.CommandLine; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using Microsoft.Build.Locator; + +namespace Saritasa.Tools.CodeAnalyzers.Benchmarks; + +/// +/// Program class. +/// +internal class Program +{ + /// + /// Entry point. + /// + public static async Task Main(string[] args) + { + var testProjectPathOption = new Option("--testProjectPath") + { + Description = "Path to the test project for code analyzer benchmarks.", + Required = true + }; + + var rootCommand = new RootCommand(); + rootCommand.Options.Add(testProjectPathOption); + rootCommand.TreatUnmatchedTokensAsErrors = false; + + var parseResult = rootCommand.Parse(args); + if (parseResult.Errors.Count == 0 && parseResult.GetValue(testProjectPathOption) is string testProjectPath) + { + CodeAnalyzersBenchmarkSettings.TestProjectPath = testProjectPath; + + MSBuildLocator.RegisterDefaults(); + + var config = ManualConfig.CreateEmpty() + // InProcess is required because MSBuildLocator.RegisterDefaults() registers an + // assembly resolver in the current AppDomain. The default out-of-process toolchain + // spawns a child process with a BenchmarkDotNet-generated Main that never calls + // MSBuildLocator, so MSBuild assemblies cannot be resolved. + .AddJob(Job.Default + .WithToolchain(InProcessEmitToolchain.Instance)) + .WithOption(ConfigOptions.StopOnFirstError, true) + .AddLogger(ConsoleLogger.Default) + .WithOption(ConfigOptions.DisableLogFile, true) + .AddDiagnoser(MemoryDiagnoser.Default) + .AddExporter(new CodeAnalyzersBenchmarkExporter()) + .AddColumnProvider(DefaultColumnProviders.Instance); + + BenchmarkRunner.Run(config, parseResult.UnmatchedTokens.ToArray()); + return 0; + } + + foreach (var parseError in parseResult.Errors) + { + await Console.Error.WriteLineAsync(parseError.Message); + } + + return 1; + } +} diff --git a/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Properties/launchSettings.json b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Properties/launchSettings.json new file mode 100644 index 00000000..d4e2d689 --- /dev/null +++ b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Properties/launchSettings.json @@ -0,0 +1,9 @@ +{ + "profiles": { + "Saritasa.Tools.CodeAnalyzers.Benchmarks": { + "workingDirectory": "$(ProjectDir)", + "commandName": "Project", + "commandLineArgs": "--testProjectPath \"$(SolutionDir)Saritasa.Tools.sln\" --artifacts artifacts" + } + } +} diff --git a/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/README.md b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/README.md new file mode 100644 index 00000000..d9275220 --- /dev/null +++ b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/README.md @@ -0,0 +1,44 @@ +# Saritasa.Tools.CodeAnalyzers.Benchmarks + +A BenchmarkDotNet-based performance benchmarking tool for Roslyn diagnostic analyzers defined in `Saritasa.Tools.CodeAnalyzers`. + +## Overview + +The tool opens a target .NET solution via MSBuild, compiles each project in it, and measures how long every `DiagnosticAnalyzer` takes to analyze those compilations. Analyzers are discovered automatically through reflection — no changes to benchmark code are required when a new analyzer is added. + +Results are exported as indented JSON files into an `--artifacts` directory using a custom exporter that uses the analyzer class name instead of the benchmark method name for readability. + +## Arguments + +| Argument | Required | Description | +|---------------------|----------|-----------------------------------------------------| +| `--testProjectPath` | Yes | Path to the `.sln` file to analyze | +| `--artifacts` | No | Output directory for benchmark result JSON files | + +Additional BenchmarkDotNet arguments (e.g. `--filter`) can be appended after the required ones. + +## Running + +### Option 1 — Launch profile (VS Code / Visual Studio) + +Open [Properties/launchSettings.json](Properties/launchSettings.json) and update `--testProjectPath` to point to the solution you want to benchmark: + +```json +"commandLineArgs": "--testProjectPath \"C:\\path\\to\\Your.sln\" --artifacts artifacts" +``` + +Then start the project using the **Saritasa.Tools.CodeAnalyzers.Benchmarks** launch profile. + +### Option 2 — Command line + +Run from the project directory (must use `Release` configuration): + +```bash +dotnet run -c Release -- --testProjectPath "C:\\path\\to\\Your.sln" --artifacts artifacts +``` + +> **Note:** The `--` separator is required to distinguish application arguments from `dotnet run` arguments. + +## Output + +Benchmark results are written as JSON files to the directory specified by `--artifacts`. \ No newline at end of file diff --git a/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Saritasa.Tools.CodeAnalyzers.Benchmarks.csproj b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Saritasa.Tools.CodeAnalyzers.Benchmarks.csproj new file mode 100644 index 00000000..818a403d --- /dev/null +++ b/test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Saritasa.Tools.CodeAnalyzers.Benchmarks.csproj @@ -0,0 +1,26 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + + + + + + + +