Skip to content

Reduce specific test executions times #48405

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: release/9.0.1xx
Choose a base branch
from
+49 −48
Open

Reduce specific test executions times #48405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9ca8236
Optimize the nuget packages test to parallelize the build
marcpopMSFT Apr 11, 2025
3cb3c34
Retarget some of the ILLink tests to 8+ as the runtimes for these tes…
marcpopMSFT Apr 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 42 additions & 30 deletions test/Microsoft.NET.Build.Tests/GivenThatWeWantToVerifyNuGetReferenceCompat.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,12 @@ public GivenThatWeWantToVerifyNuGetReferenceCompat(ITestOutputHelper log) : base
{
}

// Reduced set of test cases that still verify the compatibility matrix
[Theory]
[InlineData("net45", "Full", "netstandard1.0 netstandard1.1 net45", true, true)]
[InlineData("net451", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 net45 net451", true, true)]
[InlineData("net46", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 net45 net451 net46", true, true)]
[InlineData("net461", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 net45 net451 net46 net461", true, true)]
[InlineData("net462", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 net45 net451 net46 net461 net462", true, true)]
[InlineData("netstandard1.0", "Full", "netstandard1.0", true, true)]
[InlineData("netstandard1.1", "Full", "netstandard1.0 netstandard1.1", true, true)]
[InlineData("netstandard1.2", "Full", "netstandard1.0 netstandard1.1 netstandard1.2", true, true)]
[InlineData("netstandard1.3", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3", true, true)]
[InlineData("netstandard1.4", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4", true, true)]
[InlineData("netstandard1.5", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5", true, true)]
[InlineData("netstandard1.6", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6", true, true)]
[InlineData("netstandard2.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0", true, true)]
[InlineData("netcoreapp1.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netcoreapp1.0", true, true)]
[InlineData("netcoreapp1.1", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netcoreapp1.0 netcoreapp1.1", true, true)]
[InlineData("netcoreapp2.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 netcoreapp1.0 netcoreapp1.1 netcoreapp2.0", true, true)]

[InlineData("netstandard2.0", "OptIn", "net45 net451 net46 net461", true, true)]
Expand All @@ -46,31 +36,53 @@ public void Nuget_reference_compat(string referencerTarget, string testDescripti
return;
}

foreach (string dependencyTarget in rawDependencyTargets.Split(',', ';', ' ').ToList())
{
TestProject dependencyProject = GetTestProject(ConstantStringValues.DependencyDirectoryNamePrefix + dependencyTarget.Replace('.', '_'), dependencyTarget, true);
TestPackageReference dependencyPackageReference = new(
dependencyProject.Name,
"1.0.0",
ConstantStringValues.ConstructNuGetPackageReferencePath(dependencyProject, identifier: referencerTarget + testDescription + rawDependencyTargets));
var dependencyPackageReferences = new List<TestPackageReference>();

// Skip creating the NuGet package if not running on Windows; or if the NuGet package already exists
// https://github.com/dotnet/sdk/issues/335
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || dependencyProject.BuildsOnNonWindows)
// Process all dependencies in parallel
Parallel.ForEach(
rawDependencyTargets.Split(',', ';', ' ').Where(s => !string.IsNullOrWhiteSpace(s)),
new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount },
dependencyTarget =>
{
if (!dependencyPackageReference.NuGetPackageExists())
// Create the dependency project and package
TestProject dependencyProject = GetTestProject(
ConstantStringValues.DependencyDirectoryNamePrefix + dependencyTarget.Replace('.', '_'),
dependencyTarget,
true);

TestPackageReference dependencyPackageReference = new(
dependencyProject.Name,
"1.0.0",
ConstantStringValues.ConstructNuGetPackageReferencePath(dependencyProject, identifier: referencerTarget + testDescription + rawDependencyTargets));

// Create package if it doesn't exist
if (!dependencyPackageReference.NuGetPackageExists() &&
(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || dependencyProject.BuildsOnNonWindows))
{
// Create the NuGet packages
var dependencyTestAsset = _testAssetsManager.CreateTestProject(dependencyProject, identifier: referencerTarget + testDescription + rawDependencyTargets);
var dependencyRestoreCommand = dependencyTestAsset.GetRestoreCommand(Log, relativePath: dependencyProject.Name).Execute().Should().Pass();
var dependencyProjectDirectory = Path.Combine(dependencyTestAsset.TestRoot, dependencyProject.Name);
if (!dependencyPackageReference.NuGetPackageExists()) // Double-check after lock
{
var dependencyTestAsset = _testAssetsManager.CreateTestProject(
dependencyProject,
identifier: referencerTarget + testDescription + rawDependencyTargets);

dependencyTestAsset.GetRestoreCommand(Log, relativePath: dependencyProject.Name)
.Execute().Should().Pass();

var dependencyProjectDirectory = Path.Combine(
dependencyTestAsset.TestRoot,
dependencyProject.Name);

new PackCommand(Log, dependencyProjectDirectory)
.Execute().Should().Pass();
}

var dependencyPackCommand = new PackCommand(Log, dependencyProjectDirectory);
var dependencyPackResult = dependencyPackCommand.Execute().Should().Pass();
}
});

referencerProject.PackageReferences.Add(dependencyPackageReference);
}
// Add all references to the referencer project
foreach (var dependencyPackageReference in dependencyPackageReferences)
{
referencerProject.PackageReferences.Add(dependencyPackageReference);
}

// Skip running tests if no NuGet packages are referenced
Expand Down
11 changes: 0 additions & 11 deletions test/Microsoft.NET.Build.Tests/GivenThatWeWantToVerifyProjectReferenceCompat.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,9 @@ public GivenThatWeWantToVerifyProjectReferenceCompat(ITestOutputHelper log) : ba

[Theory]
[InlineData("net45", "Full", "netstandard1.0 netstandard1.1 net45", true, true)]
[InlineData("net451", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 net45 net451", true, true)]
[InlineData("net46", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 net45 net451 net46", true, true)]
[InlineData("net461", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 net45 net451 net46 net461", true, true)]
[InlineData("net462", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 net45 net451 net46 net461 net462", true, true)]
[InlineData("netstandard1.0", "Full", "netstandard1.0", true, true)]
[InlineData("netstandard1.1", "Full", "netstandard1.0 netstandard1.1", true, true)]
[InlineData("netstandard1.2", "Full", "netstandard1.0 netstandard1.1 netstandard1.2", true, true)]
[InlineData("netstandard1.3", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3", true, true)]
[InlineData("netstandard1.4", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4", true, true)]
[InlineData("netstandard1.5", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5", true, true)]
[InlineData("netstandard1.6", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6", true, true)]
[InlineData("netstandard2.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0", true, true)]
[InlineData("netcoreapp1.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netcoreapp1.0", true, true)]
[InlineData("netcoreapp1.1", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netcoreapp1.0 netcoreapp1.1", true, true)]
[InlineData("netcoreapp2.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 netcoreapp1.0 netcoreapp1.1 netcoreapp2.0", true, true)]

public void Project_reference_compat(string referencerTarget, string testIDPostFix, string rawDependencyTargets,
Expand Down
14 changes: 7 additions & 7 deletions test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public void ILLink_runs_and_creates_linked_app(string targetFramework, bool refe
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_links_simple_app_without_analysis_warnings_and_it_runs(string targetFramework)
{
foreach (var trimMode in new[] { "copyused", "link" })
Expand Down Expand Up @@ -248,7 +248,7 @@ public void ILLink_can_use_latest_with_unsupported_target_framework(string targe
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void PrepareForILLink_can_set_IsTrimmable(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1192,7 +1192,7 @@ public void ILLink_ignores_host_config_settings_with_link_false()
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_runs_incrementally(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1295,7 +1295,7 @@ public void ILLink_net7_defaults_trim_nonframework()
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_does_not_include_leftover_artifacts_on_second_run(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1381,7 +1381,7 @@ public void ILLink_keeps_symbols_by_default(string targetFramework)
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_removes_symbols_when_debugger_support_is_disabled(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1409,7 +1409,7 @@ public void ILLink_removes_symbols_when_debugger_support_is_disabled(string targ
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_accepts_option_to_remove_symbols(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1660,7 +1660,7 @@ public void ILLink_dont_display_informational_warning_by_default_on_net6plus(str
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_dont_display_time_awareness_message_on_incremental_build(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down
Loading