Skip to content

Reduce specific test executions times #48405

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 10, 2025
Merged

Reduce specific test executions times #48405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9ca8236
Optimize the nuget packages test to parallelize the build
marcpopMSFT Apr 11, 2025
3cb3c34
Retarget some of the ILLink tests to 8+ as the runtimes for these tes…
marcpopMSFT Apr 11, 2025
539fc44
Respond to PR feedback
marcpopMSFT May 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 41 additions & 30 deletions test/Microsoft.NET.Build.Tests/GivenThatWeWantToVerifyNuGetReferenceCompat.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,9 @@ public GivenThatWeWantToVerifyNuGetReferenceCompat(ITestOutputHelper log) : base

[Theory]
[InlineData("net45", "Full", "netstandard1.0 netstandard1.1 net45", true, true)]
[InlineData("net451", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 net45 net451", true, true)]
[InlineData("net46", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 net45 net451 net46", true, true)]
[InlineData("net461", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 net45 net451 net46 net461", true, true)]
[InlineData("net462", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 net45 net451 net46 net461 net462", true, true)]
[InlineData("netstandard1.0", "Full", "netstandard1.0", true, true)]
[InlineData("netstandard1.1", "Full", "netstandard1.0 netstandard1.1", true, true)]
[InlineData("netstandard1.2", "Full", "netstandard1.0 netstandard1.1 netstandard1.2", true, true)]
[InlineData("netstandard1.3", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3", true, true)]
[InlineData("netstandard1.4", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4", true, true)]
[InlineData("netstandard1.5", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5", true, true)]
[InlineData("netstandard1.6", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6", true, true)]
[InlineData("netstandard2.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0", true, true)]
[InlineData("netcoreapp1.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netcoreapp1.0", true, true)]
[InlineData("netcoreapp1.1", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netcoreapp1.0 netcoreapp1.1", true, true)]
[InlineData("netcoreapp2.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 netcoreapp1.0 netcoreapp1.1 netcoreapp2.0", true, true)]

[InlineData("netstandard2.0", "OptIn", "net45 net451 net46 net461", true, true)]
Expand All @@ -46,31 +35,53 @@ public void Nuget_reference_compat(string referencerTarget, string testDescripti
return;
}

foreach (string dependencyTarget in rawDependencyTargets.Split(',', ';', ' ').ToList())
{
TestProject dependencyProject = GetTestProject(ConstantStringValues.DependencyDirectoryNamePrefix + dependencyTarget.Replace('.', '_'), dependencyTarget, true);
TestPackageReference dependencyPackageReference = new(
dependencyProject.Name,
"1.0.0",
ConstantStringValues.ConstructNuGetPackageReferencePath(dependencyProject, identifier: referencerTarget + testDescription + rawDependencyTargets));
var dependencyPackageReferences = new List<TestPackageReference>();

// Skip creating the NuGet package if not running on Windows; or if the NuGet package already exists
// https://github.com/dotnet/sdk/issues/335
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || dependencyProject.BuildsOnNonWindows)
// Process all dependencies in parallel
Parallel.ForEach(
rawDependencyTargets.Split(',', ';', ' ').Where(s => !string.IsNullOrWhiteSpace(s)),
new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount },
dependencyTarget =>
{
if (!dependencyPackageReference.NuGetPackageExists())
// Create the dependency project and package
TestProject dependencyProject = GetTestProject(
ConstantStringValues.DependencyDirectoryNamePrefix + dependencyTarget.Replace('.', '_'),
dependencyTarget,
true);

TestPackageReference dependencyPackageReference = new(
dependencyProject.Name,
"1.0.0",
ConstantStringValues.ConstructNuGetPackageReferencePath(dependencyProject, identifier: referencerTarget + testDescription + rawDependencyTargets));

// Create package if it doesn't exist
if (!dependencyPackageReference.NuGetPackageExists() &&
(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || dependencyProject.BuildsOnNonWindows))
{
// Create the NuGet packages
var dependencyTestAsset = _testAssetsManager.CreateTestProject(dependencyProject, identifier: referencerTarget + testDescription + rawDependencyTargets);
var dependencyRestoreCommand = dependencyTestAsset.GetRestoreCommand(Log, relativePath: dependencyProject.Name).Execute().Should().Pass();
var dependencyProjectDirectory = Path.Combine(dependencyTestAsset.TestRoot, dependencyProject.Name);
if (!dependencyPackageReference.NuGetPackageExists())
{
var dependencyTestAsset = _testAssetsManager.CreateTestProject(
dependencyProject,
identifier: referencerTarget + testDescription + rawDependencyTargets);

dependencyTestAsset.GetRestoreCommand(Log, relativePath: dependencyProject.Name)
.Execute().Should().Pass();

var dependencyProjectDirectory = Path.Combine(
dependencyTestAsset.TestRoot,
dependencyProject.Name);

new PackCommand(Log, dependencyProjectDirectory)
.Execute().Should().Pass();
}

var dependencyPackCommand = new PackCommand(Log, dependencyProjectDirectory);
var dependencyPackResult = dependencyPackCommand.Execute().Should().Pass();
}
});

referencerProject.PackageReferences.Add(dependencyPackageReference);
}
// Add all references to the referencer project
foreach (var dependencyPackageReference in dependencyPackageReferences)
{
referencerProject.PackageReferences.Add(dependencyPackageReference);
}

// Skip running tests if no NuGet packages are referenced
Expand Down
11 changes: 0 additions & 11 deletions test/Microsoft.NET.Build.Tests/GivenThatWeWantToVerifyProjectReferenceCompat.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,9 @@ public GivenThatWeWantToVerifyProjectReferenceCompat(ITestOutputHelper log) : ba

[Theory]
[InlineData("net45", "Full", "netstandard1.0 netstandard1.1 net45", true, true)]
[InlineData("net451", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 net45 net451", true, true)]
[InlineData("net46", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 net45 net451 net46", true, true)]
[InlineData("net461", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 net45 net451 net46 net461", true, true)]
[InlineData("net462", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 net45 net451 net46 net461 net462", true, true)]
[InlineData("netstandard1.0", "Full", "netstandard1.0", true, true)]
[InlineData("netstandard1.1", "Full", "netstandard1.0 netstandard1.1", true, true)]
[InlineData("netstandard1.2", "Full", "netstandard1.0 netstandard1.1 netstandard1.2", true, true)]
[InlineData("netstandard1.3", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3", true, true)]
[InlineData("netstandard1.4", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4", true, true)]
[InlineData("netstandard1.5", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5", true, true)]
[InlineData("netstandard1.6", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6", true, true)]
[InlineData("netstandard2.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0", true, true)]
[InlineData("netcoreapp1.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netcoreapp1.0", true, true)]
[InlineData("netcoreapp1.1", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netcoreapp1.0 netcoreapp1.1", true, true)]
[InlineData("netcoreapp2.0", "Full", "netstandard1.0 netstandard1.1 netstandard1.2 netstandard1.3 netstandard1.4 netstandard1.5 netstandard1.6 netstandard2.0 netcoreapp1.0 netcoreapp1.1 netcoreapp2.0", true, true)]

public void Project_reference_compat(string referencerTarget, string testIDPostFix, string rawDependencyTargets,
Expand Down
14 changes: 7 additions & 7 deletions test/Microsoft.NET.Publish.Tests/GivenThatWeWantToRunILLink.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public void ILLink_runs_and_creates_linked_app(string targetFramework, bool refe
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe just remove the unsupported TFMs from SupportedTfms?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@agocke I interpreted "SupprotedTfms" as being supported for the feature, not in support as it goes back to netcoreapp3.1 and has been for a while. These are your tests so happy to change what supported means here to be the same as net8plus for the moment.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I added that I actually meant it as "the SDK is in support", but I can clean this up later

public void ILLink_links_simple_app_without_analysis_warnings_and_it_runs(string targetFramework)
{
foreach (var trimMode in new[] { "copyused", "link" })
Expand Down Expand Up @@ -248,7 +248,7 @@ public void ILLink_can_use_latest_with_unsupported_target_framework(string targe
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void PrepareForILLink_can_set_IsTrimmable(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1192,7 +1192,7 @@ public void ILLink_ignores_host_config_settings_with_link_false()
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_runs_incrementally(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1295,7 +1295,7 @@ public void ILLink_net7_defaults_trim_nonframework()
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_does_not_include_leftover_artifacts_on_second_run(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1381,7 +1381,7 @@ public void ILLink_keeps_symbols_by_default(string targetFramework)
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_removes_symbols_when_debugger_support_is_disabled(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1409,7 +1409,7 @@ public void ILLink_removes_symbols_when_debugger_support_is_disabled(string targ
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_accepts_option_to_remove_symbols(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down Expand Up @@ -1660,7 +1660,7 @@ public void ILLink_dont_display_informational_warning_by_default_on_net6plus(str
}

[RequiresMSBuildVersionTheory("17.0.0.32901")]
[MemberData(nameof(SupportedTfms), MemberType = typeof(PublishTestUtils))]
[MemberData(nameof(Net8Plus), MemberType = typeof(PublishTestUtils))]
public void ILLink_dont_display_time_awareness_message_on_incremental_build(string targetFramework)
{
var projectName = "HelloWorld";
Expand Down
Loading