Skip to content

Commit 28acc55

Browse files
AnastasiaLyutovaAnastasiaLyutova
authored
Merge pull request #107 from saritasa-nest/feature/SN-959-Create-performance-tests-for-analyzers
Feature/sn 959 create performance tests for analyzers
2 parents 421664d + 47775af commit 28acc55

9 files changed

Lines changed: 405 additions & 0 deletions

‎.github/workflows/run-code-analyzers-benchmarks.yml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
# ==============================================================================
2+
# Code Analyzers Performance Comparison
3+
#
4+
# Triggered on pull requests that modify Saritasa.Tools.CodeAnalyzers project.
5+
#
6+
# How it works:
7+
# 1. Checks out the PR branch and runs benchmarks against an external test project.
8+
# 2. Switches only the Saritasa.Tools.CodeAnalyzers source to the base branch (master),
9+
# keeping the benchmark code and test project unchanged.
10+
# 3. Runs benchmarks again to get the baseline results.
11+
# 4. Compares the two runs and fails if mean time or allocations regress
12+
# beyond the configured thresholds (THRESHOLD_MEAN / THRESHOLD_ALLOCATION).
13+
# 5. Posts the comparison report as a sticky PR comment.
14+
#
15+
# More info about tool used to analyze and compare at https://github.com/TechNobre/PowerUtils.BenchmarkDotnet.Reporter.
16+
# ==============================================================================
17+
name: Code Analyzers Performance Comparison
18+
19+
on:
20+
pull_request:
21+
branches:
22+
- master
23+
24+
paths:
25+
- 'src/Saritasa.Tools.CodeAnalyzers/**'
26+
27+
# Allows you to run this workflow manually from the Actions tab
28+
workflow_dispatch:
29+
30+
env:
31+
PATH_BENCHMARKS_PROJECT: 'test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Saritasa.Tools.CodeAnalyzers.Benchmarks.csproj'
32+
DIR_BASELINE_REPORTS: './artifacts-baseline'
33+
DIR_TARGET_REPORTS: './artifacts-target'
34+
PATH_REPORT_RESULT: './BenchmarkReporter/benchmark-comparison-report.md'
35+
TEST_PROJECT_FOLDER: './testProject'
36+
PATH_TO_TEST_SOLUTION: 'src/Saritasa.NetForge.slnx'
37+
THRESHOLD_MEAN: 50%
38+
THRESHOLD_ALLOCATION: 50%
39+
40+
jobs:
41+
compare-code-analyzers-performance:
42+
runs-on: ubuntu-latest
43+
44+
steps:
45+
# Ensure a clean test project directory before checkout.
46+
- name: Clean up target directory
47+
run: rm -rf ${{ env.TEST_PROJECT_FOLDER }}
48+
49+
# Checkout the PR branch so we can benchmark the proposed changes.
50+
- name: Checkout PR Branch (Current Changes)
51+
uses: actions/checkout@v4
52+
53+
# Checkout the external test project used as input for the analyzers.
54+
- name: Checkout External Repository
55+
id: external_checkout
56+
uses: actions/checkout@v4
57+
with:
58+
repository: 'saritasa-nest/saritasa-forge-admin'
59+
path: ${{ env.TEST_PROJECT_FOLDER }}
60+
persist-credentials: false
61+
ssh-strict: false
62+
set-safe-directory: false
63+
fetch-depth: 1
64+
65+
# Pin the test project to a known commit for reproducible benchmark input.
66+
- name: Reset to specific commit
67+
uses: actions/checkout@v4
68+
with:
69+
repository: 'saritasa-nest/saritasa-forge-admin'
70+
ref: dc2d1ed5d58be20e709de5ba884beeaba0c6a733
71+
path: ${{ env.TEST_PROJECT_FOLDER }}
72+
73+
- name: Setup .NET
74+
uses: actions/setup-dotnet@v4
75+
with:
76+
dotnet-version: '10.0.x'
77+
78+
# Benchmark the PR branch. sudo -E preserves the environment so this allows
79+
# BenchmarkDotNet to raise the process priority for more stable measurements.
80+
- name: Run PR Benchmarks
81+
run: |
82+
sudo -E dotnet run --no-launch-profile -c Release \
83+
--project ${{ env.PATH_BENCHMARKS_PROJECT }} -- \
84+
--artifacts ${{ env.DIR_TARGET_REPORTS }} \
85+
--testProjectPath ${{ env.TEST_PROJECT_FOLDER }}/${{ env.PATH_TO_TEST_SOLUTION }}
86+
87+
# Swap the Saritasa.Tools.CodeAnalyzers code to the base branch while keeping the test project intact.
88+
# The directory is removed first to ensure files added in the PR (not present in base) are deleted.
89+
# Without this, `git checkout FETCH_HEAD -- <path>` only updates existing files and leaves new ones behind.
90+
- name: Switch Saritasa.Tools.CodeAnalyzers to Base Branch
91+
run: |
92+
git fetch origin ${{ github.base_ref }} --depth=1
93+
sudo rm -rf src/Saritasa.Tools.CodeAnalyzers/
94+
git checkout FETCH_HEAD -- src/Saritasa.Tools.CodeAnalyzers/
95+
git status
96+
97+
# Benchmark the base branch with the same test project for a fair comparison.
98+
- name: Run Base Benchmarks
99+
run: |
100+
sudo -E dotnet run --no-launch-profile -c Release \
101+
--project ${{ env.PATH_BENCHMARKS_PROJECT }} -- \
102+
--artifacts ${{ env.DIR_BASELINE_REPORTS }} \
103+
--testProjectPath ${{ env.TEST_PROJECT_FOLDER }}/${{ env.PATH_TO_TEST_SOLUTION }}
104+
105+
- name: Install lib for benchmarks comparison
106+
run: sudo -E dotnet tool install --global PowerUtils.BenchmarkDotnet.Reporter
107+
108+
- name: Run benchmarks compare
109+
run: |
110+
pbreporter compare \
111+
-b ${{ env.DIR_BASELINE_REPORTS }}/results \
112+
-t ${{ env.DIR_TARGET_REPORTS }}/results \
113+
-f markdown \
114+
-tm ${{ env.THRESHOLD_MEAN }} \
115+
-ta ${{ env.THRESHOLD_ALLOCATION }} \
116+
-ft -fw
117+
118+
- name: Publish benchmark report in Summary
119+
run: cat ${{ env.PATH_REPORT_RESULT }} > $GITHUB_STEP_SUMMARY
120+
121+
- name: Add compare benchmark report in PR Comment
122+
uses: marocchino/sticky-pull-request-comment@v2
123+
if: github.event_name == 'pull_request'
124+
with:
125+
header: compare-benchmark-report
126+
hide_and_recreate: true
127+
hide_classify: "OUTDATED"
128+
path: ${{ env.PATH_REPORT_RESULT }}

‎Saritasa.Tools.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Saritasa.Tools.CodeAnalyzer
4646
EndProject
4747
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Saritasa.Tools.CodeAnalyzers.Tests", "test\Saritasa.Tools.CodeAnalyzers.Tests\Saritasa.Tools.CodeAnalyzers.Tests.csproj", "{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}"
4848
EndProject
49+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Saritasa.Tools.CodeAnalyzers.Benchmarks", "test\Saritasa.Tools.CodeAnalyzers.Benchmarks\Saritasa.Tools.CodeAnalyzers.Benchmarks.csproj", "{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}"
50+
EndProject
4951
Global
5052
GlobalSection(SolutionConfigurationPlatforms) = preSolution
5153
Debug|Any CPU = Debug|Any CPU
@@ -112,6 +114,10 @@ Global
112114
{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}.Debug|Any CPU.Build.0 = Debug|Any CPU
113115
{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}.Release|Any CPU.ActiveCfg = Release|Any CPU
114116
{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2}.Release|Any CPU.Build.0 = Release|Any CPU
117+
{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
118+
{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Debug|Any CPU.Build.0 = Debug|Any CPU
119+
{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Release|Any CPU.ActiveCfg = Release|Any CPU
120+
{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F}.Release|Any CPU.Build.0 = Release|Any CPU
115121
EndGlobalSection
116122
GlobalSection(SolutionProperties) = preSolution
117123
HideSolutionNode = FALSE
@@ -132,6 +138,7 @@ Global
132138
{31FDE475-97E3-49C0-85BA-FD782061514B} = {635DB5B1-86B3-4F5B-97A6-04C71D672296}
133139
{29B003C4-F226-47AC-B591-3C563E6667AB} = {45D13EC5-88F3-403A-8148-0812C2796062}
134140
{908D4D8F-B7E3-46DC-8517-A8FCE7AF29E2} = {635DB5B1-86B3-4F5B-97A6-04C71D672296}
141+
{03FB7D6A-2A23-4BB8-A2C3-87A5B791832F} = {635DB5B1-86B3-4F5B-97A6-04C71D672296}
135142
EndGlobalSection
136143
GlobalSection(ExtensibilityGlobals) = postSolution
137144
SolutionGuid = {F298C89B-4162-49D5-B517-5306BD58DB59}

‎test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkExporter.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
using BenchmarkDotNet.Exporters.Json;
2+
using BenchmarkDotNet.Reports;
3+
4+
namespace Saritasa.Tools.CodeAnalyzers.Benchmarks;
5+
6+
/// <summary>
7+
/// A custom JSON exporter that replaces the benchmark method name with the value of its first parameter
8+
/// for a more convenient display of code analyzer test results.
9+
/// </summary>
10+
public class CodeAnalyzersBenchmarkExporter : JsonExporterBase
11+
{
12+
/// <summary>
13+
/// Constructor.
14+
/// </summary>
15+
public CodeAnalyzersBenchmarkExporter() : base(indentJson: true, excludeMeasurements: true)
16+
{
17+
}
18+
19+
/// <inheritdoc/>
20+
protected override IReadOnlyDictionary<string, object> GetDataToSerialize(BenchmarkReport report)
21+
{
22+
var dict = base.GetDataToSerialize(report);
23+
24+
var firstParam = report.BenchmarkCase.Parameters.Items.FirstOrDefault();
25+
var methodName = firstParam?.Value?.ToString() ?? report.BenchmarkCase.Descriptor.WorkloadMethod.Name;
26+
27+
var copy = dict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
28+
29+
copy["Method"] = methodName;
30+
copy["MethodTitle"] = methodName;
31+
32+
return copy;
33+
}
34+
}

‎test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarkSettings.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
namespace Saritasa.Tools.CodeAnalyzers.Benchmarks;
2+
3+
/// <summary>
4+
/// Benchmark run settings.
5+
/// </summary>
6+
internal static class CodeAnalyzersBenchmarkSettings
7+
{
8+
/// <summary>
9+
/// Path to the test project solution file passed via --testProjectPath argument.
10+
/// </summary>
11+
public static string TestProjectPath { get; set; } = string.Empty;
12+
}

‎test/Saritasa.Tools.CodeAnalyzers.Benchmarks/CodeAnalyzersBenchmarks.cs‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
using BenchmarkDotNet.Attributes;
2+
using Microsoft.CodeAnalysis;
3+
using Microsoft.CodeAnalysis.Diagnostics;
4+
using Microsoft.CodeAnalysis.MSBuild;
5+
using Saritasa.Tools.CodeAnalyzers.Analyzers;
6+
7+
namespace Saritasa.Tools.CodeAnalyzers.Benchmarks;
8+
9+
/// <summary>
10+
/// Wraps a DiagnosticAnalyzer to provide a short display name for BenchmarkDotNet.
11+
/// </summary>
12+
public class AnalyzerParam(DiagnosticAnalyzer analyzer)
13+
{
14+
/// <summary>
15+
/// Analyzer.
16+
/// </summary>
17+
public DiagnosticAnalyzer Analyzer { get; } = analyzer;
18+
19+
/// <inheritdoc/>
20+
public override string ToString() => Analyzer.GetType().Name;
21+
}
22+
23+
/// <summary>
24+
/// Benchmarks for Roslyn diagnostic analyzers.
25+
/// Analyzers are discovered dynamically via reflection from Saritasa.Tools.CodeAnalyzers assembly,
26+
/// so no changes to this file are needed when a new analyzer is added.
27+
/// </summary>
28+
public class CodeAnalyzersBenchmarks
29+
{
30+
private static readonly List<Compilation> compilations = new();
31+
32+
/// <summary>
33+
/// Analyzer source.
34+
/// </summary>
35+
public static IEnumerable<AnalyzerParam> AnalyzerSource { get; } =
36+
typeof(LineLengthAnalyzer).Assembly
37+
.GetTypes()
38+
.Where(t => !t.IsAbstract && typeof(DiagnosticAnalyzer).IsAssignableFrom(t))
39+
.Select(t => new AnalyzerParam((DiagnosticAnalyzer)Activator.CreateInstance(t)!))
40+
.ToList();
41+
42+
static CodeAnalyzersBenchmarks()
43+
{
44+
using var workspace = MSBuildWorkspace.Create();
45+
46+
workspace.RegisterWorkspaceFailedHandler(args =>
47+
{
48+
Console.WriteLine($"[MSBuild] {args.Diagnostic.Message}");
49+
});
50+
51+
var solution = workspace.OpenSolutionAsync(CodeAnalyzersBenchmarkSettings.TestProjectPath).GetAwaiter().GetResult();
52+
53+
foreach (var project in solution.Projects)
54+
{
55+
var compilation = project.GetCompilationAsync().GetAwaiter().GetResult();
56+
if (compilation == null)
57+
{
58+
continue;
59+
}
60+
compilations.Add(compilation);
61+
}
62+
}
63+
64+
/// <summary>
65+
/// Runs a single analyzer against all compiled projects.
66+
/// Generates one benchmark case per analyzer found in AnalyzerSource.
67+
/// </summary>
68+
[Benchmark]
69+
[ArgumentsSource(nameof(AnalyzerSource))]
70+
public async Task RunAnalyzer(AnalyzerParam param)
71+
{
72+
foreach (var compilation in compilations)
73+
{
74+
// A new instance must be created each iteration: CompilationWithAnalyzers caches
75+
// results internally, so reusing it would measure cache retrieval, not actual analysis.
76+
var compilationWithAnalyzers = compilation.WithAnalyzers([param.Analyzer]);
77+
_ = await compilationWithAnalyzers.GetAnalyzerDiagnosticsAsync();
78+
}
79+
}
80+
}

‎test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Program.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using System.CommandLine;
2+
using BenchmarkDotNet.Columns;
3+
using BenchmarkDotNet.Configs;
4+
using BenchmarkDotNet.Diagnosers;
5+
using BenchmarkDotNet.Jobs;
6+
using BenchmarkDotNet.Loggers;
7+
using BenchmarkDotNet.Running;
8+
using BenchmarkDotNet.Toolchains.InProcess.Emit;
9+
using Microsoft.Build.Locator;
10+
11+
namespace Saritasa.Tools.CodeAnalyzers.Benchmarks;
12+
13+
/// <summary>
14+
/// Program class.
15+
/// </summary>
16+
internal class Program
17+
{
18+
/// <summary>
19+
/// Entry point.
20+
/// </summary>
21+
public static async Task<int> Main(string[] args)
22+
{
23+
var testProjectPathOption = new Option<string>("--testProjectPath")
24+
{
25+
Description = "Path to the test project for code analyzer benchmarks.",
26+
Required = true
27+
};
28+
29+
var rootCommand = new RootCommand();
30+
rootCommand.Options.Add(testProjectPathOption);
31+
rootCommand.TreatUnmatchedTokensAsErrors = false;
32+
33+
var parseResult = rootCommand.Parse(args);
34+
if (parseResult.Errors.Count == 0 && parseResult.GetValue(testProjectPathOption) is string testProjectPath)
35+
{
36+
CodeAnalyzersBenchmarkSettings.TestProjectPath = testProjectPath;
37+
38+
MSBuildLocator.RegisterDefaults();
39+
40+
var config = ManualConfig.CreateEmpty()
41+
// InProcess is required because MSBuildLocator.RegisterDefaults() registers an
42+
// assembly resolver in the current AppDomain. The default out-of-process toolchain
43+
// spawns a child process with a BenchmarkDotNet-generated Main that never calls
44+
// MSBuildLocator, so MSBuild assemblies cannot be resolved.
45+
.AddJob(Job.Default
46+
.WithToolchain(InProcessEmitToolchain.Instance))
47+
.WithOption(ConfigOptions.StopOnFirstError, true)
48+
.AddLogger(ConsoleLogger.Default)
49+
.WithOption(ConfigOptions.DisableLogFile, true)
50+
.AddDiagnoser(MemoryDiagnoser.Default)
51+
.AddExporter(new CodeAnalyzersBenchmarkExporter())
52+
.AddColumnProvider(DefaultColumnProviders.Instance);
53+
54+
BenchmarkRunner.Run<CodeAnalyzersBenchmarks>(config, parseResult.UnmatchedTokens.ToArray());
55+
return 0;
56+
}
57+
58+
foreach (var parseError in parseResult.Errors)
59+
{
60+
await Console.Error.WriteLineAsync(parseError.Message);
61+
}
62+
63+
return 1;
64+
}
65+
}

‎test/Saritasa.Tools.CodeAnalyzers.Benchmarks/Properties/launchSettings.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"profiles": {
3+
"Saritasa.Tools.CodeAnalyzers.Benchmarks": {
4+
"workingDirectory": "$(ProjectDir)",
5+
"commandName": "Project",
6+
"commandLineArgs": "--testProjectPath \"$(SolutionDir)Saritasa.Tools.sln\" --artifacts artifacts"
7+
}
8+
}
9+
}

0 commit comments

Comments
 (0)