Skip to content

Parallel CI jobs #1904

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
damianh merged 1 commit into from
Mar 21, 2025
Merged

Parallel CI jobs #1904

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 141 additions & 76 deletions .github/workflow-gen/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,29 @@

var products = new Product[]
{
new("aspnetcore-authentication-jwtbearer", "aspnetcore-authentication-jwtbearer.slnf", "aaj"),
new("identity-server", "identity-server.slnf", "is"),
new("bff", "bff.slnf", "bff", true)
new("aspnetcore-authentication-jwtbearer",
"aspnetcore-authentication-jwtbearer.slnf",
"aaj",
[],
[]),
new("identity-server",
"identity-server.slnf",
"is",
[],
[]),
new("bff",
"bff.slnf",
"bff",
["Bff.Tests", "Bff.Blazor.Client.UnitTests", "Bff.Blazor.UnitTests", "Bff.EntityFramework.Tests"],
["Hosts.Tests"])
};
foreach (var product in products)
{
GenerateCiWorkflow(product);
GenerateReleaseWorkflow(product);
GenerateCodeQlWorkflow(product, "38 15 * * 0");
}

GenerateTemplatesReleaseWorkflow(new Product("templates", "../artifacts/templates.csproj", "templates"));
GenerateTemplatesReleaseWorkflow(new Product("templates", "../artifacts/templates.csproj", "templates", [], []));


void GenerateCiWorkflow(Product product)
Expand All @@ -42,57 +53,158 @@ void GenerateCiWorkflow(Product product)

workflow.EnvDefaults();

var job = workflow
.Job("build")
const string VerifyFormattingJobId = "verify-formatting";
const string CodeQlJobId = "codeql";
const string PlaywrightJobId = "playwright";
const string BuildJobId = "build";

// Verify formatting
var verifyFormattingJob = workflow
.Job(VerifyFormattingJobId)
.RunEitherOnBranchOrAsPR()
.Name("Build")
.Name("Verify formatting")
.RunsOn(GitHubHostedRunners.UbuntuLatest)
.Defaults().Run("bash", product.Name)
.Job;

job.Permissions(
verifyFormattingJob.Permissions(contents: Permission.Read);

verifyFormattingJob.TimeoutMinutes(15);

verifyFormattingJob.Step()
.ActionsCheckout();

verifyFormattingJob.StepSetupDotNet();

verifyFormattingJob.StepRestore(product.Solution);

verifyFormattingJob.StepVerifyFormatting(product.Solution);

// Build
var build = workflow
.Job(BuildJobId)
.RunEitherOnBranchOrAsPR()
.Name("Build and test (unit)")
.RunsOn(GitHubHostedRunners.UbuntuLatest)
.Defaults().Run("bash", product.Name)
.Job;

build.Permissions(
actions: Permission.Read,
contents: Permission.Read,
checks: Permission.Write,
packages: Permission.Write);

job.TimeoutMinutes(15);
build.TimeoutMinutes(15);

job.Step()
build.Step()
.ActionsCheckout();

job.StepSetupDotNet();

job.StepRestore(product.Solution);
build.StepSetupDotNet();

job.StepVerifyFormatting(product.Solution);
build.StepRestore(product.Solution);

job.StepBuild(product.Solution);
build.StepBuild(product.Solution);

// Devcerts are needed because some tests run start a http server with https.
job.StepDotNetDevCerts();
build.StepDotNetDevCerts();

if (product.EnablePlaywright)
foreach (var project in product.UnitTestProjects)
{
job.StepInstallPlayWright();
build.StepTest($"test/{project}");
}

job.StepTest(product.Solution);
// Playwright
var playwrightJob = workflow
.Job(PlaywrightJobId)
.RunEitherOnBranchOrAsPR()
.Name("Playwright tests")
.RunsOn(GitHubHostedRunners.UbuntuLatest)
.Defaults().Run("bash", product.Name)
.Job;

playwrightJob.Permissions(
actions: Permission.Read,
contents: Permission.Read,
checks: Permission.Write);

playwrightJob.TimeoutMinutes(15);

playwrightJob.Step()
.ActionsCheckout();

if (product.EnablePlaywright)
if (product.PlaywrightTestProjects.Length > 0)
{
job.StepUploadPlaywrightTestTraces(product.Name);
playwrightJob.StepSetupDotNet();

playwrightJob.StepRestore(product.Solution);

playwrightJob.StepBuild(product.Solution);

playwrightJob.StepInstallPlayWright();

playwrightJob.StepDotNetDevCerts();

foreach (var project in product.PlaywrightTestProjects)
{
playwrightJob.StepTest($"test/{project}");
}

playwrightJob.StepUploadPlaywrightTestTraces(product.Name);
}

job.StepToolRestore();
// CodeQL
var codeQlJob = workflow
.Job(CodeQlJobId)
.RunEitherOnBranchOrAsPR()
.Name("CodeQL analyze")
.RunsOn(GitHubHostedRunners.UbuntuLatest)
.Defaults().Run("bash", product.Name)
.Job;

job.StepPack(product.Solution);
codeQlJob.Step()
.ActionsCheckout();

codeQlJob.StepInitializeCodeQl();

job.StepSign();
codeQlJob.StepSetupDotNet();

job.StepPushToGithub(contexts);
codeQlJob.StepRestore(product.Solution);

job.StepUploadArtifacts(product.Name);
codeQlJob.StepBuild(product.Solution);

codeQlJob.StepPerformCodeQlAnalysis();

// Pack
var packJob = workflow
.Job("pack")
.RunEitherOnBranchOrAsPR()
.Name("Pack, sign and push")
.RunsOn(GitHubHostedRunners.UbuntuLatest)
.Needs(VerifyFormattingJobId, BuildJobId, PlaywrightJobId, CodeQlJobId)
.Defaults().Run("bash", product.Name)
.Job;

packJob.Permissions(
actions: Permission.Read,
contents: Permission.Read,
packages: Permission.Write);

packJob.TimeoutMinutes(15);

packJob.Step()
.ActionsCheckout();

packJob.StepSetupDotNet();

packJob.StepToolRestore();

packJob.StepPack(product.Solution);

packJob.StepSign();

packJob.StepPushToGithub(contexts);

packJob.StepUploadArtifacts(product.Name);

var fileName = $"{product.Name}-ci";
WriteWorkflow(workflow, fileName);
Expand Down Expand Up @@ -158,53 +270,6 @@ void GenerateReleaseWorkflow(Product product)
WriteWorkflow(workflow, fileName);
}

void GenerateCodeQlWorkflow(Product system, string cronSchedule)
{
var workflow = new Workflow($"{system.Name}/codeql");
var branches = new[] { "main" };
var paths = new[] { $"{system.Name}/**" };

workflow.On
.WorkflowDispatch();
workflow.On
.Push()
.Branches(branches)
.Paths(paths);
workflow.On
.PullRequest()
.Paths(paths);
workflow.On
.Schedule(cronSchedule);

var job = workflow
.Job("analyze")
.Name("Analyze")
.RunsOn(GitHubHostedRunners.UbuntuLatest)
.Defaults().Run("bash", system.Name)
.Job;

job.Permissions(
actions: Permission.Read,
contents: Permission.Read,
securityEvents: Permission.Write);

job.Step()
.ActionsCheckout();

job.StepInitializeCodeQl();

job.StepSetupDotNet();

job.StepRestore(system.Solution);

job.StepBuild(system.Solution);

job.StepPerformCodeQlAnalysis();

var fileName = $"{system.Name}-codeql-analysis";
WriteWorkflow(workflow, fileName);
}

void GenerateTemplatesReleaseWorkflow(Product product)
{
var workflow = new Workflow($"{product.Name}/release");
Expand Down Expand Up @@ -276,4 +341,4 @@ void WriteWorkflow(Workflow workflow, string fileName)
Console.WriteLine($"Wrote workflow to {filePath}");
}

record Product(string Name, string Solution, string TagPrefix, bool EnablePlaywright = false);
record Product(string Name, string Solution, string TagPrefix, string[] UnitTestProjects, string[] PlaywrightTestProjects);
37 changes: 25 additions & 12 deletions .github/workflow-gen/StepExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,16 @@ public static void EnvDefaults(this Workflow workflow)
("DOTNET_CLI_TELEMETRY_OPTOUT", "true"));

public static void StepSetupDotNet(this Job job)
=> job.Step()
{
job.Step()
.Name("List .net sdks")
.Run("dotnet --list-sdks");

job.Step()
.Name("Setup .NET")
.ActionsSetupDotNet("3e891b0cb619bf60e2c25674b222b8940e2c1c25", ["6.0.x", "8.0.x", "9.0.103"]); // v4.1.0
.ActionsSetupDotNet("3e891b0cb619bf60e2c25674b222b8940e2c1c25", ["8.0.x", "9.0.103"]);
// v4.1.0
}

/// <summary>
/// Only run this for a main build
Expand Down Expand Up @@ -63,28 +70,33 @@ public static Step StepBuild(this Job job, string solution)
.Name("Build")
.Run($"dotnet build {solution} --no-restore -c Release");

public static void StepTest(this Job job, string solution)
public static void StepTest(this Job job, string project)
{
var logFileName = "Tests.trx";
var logFileName = $"{project}-tests.trx";
var loggingFlags = $"--logger \"console;verbosity=normal\" " +
$"--logger \"trx;LogFileName={logFileName}\" " +
$"--collect:\"XPlat Code Coverage\"";

job.Step()
.Name("Test")
.Run($"dotnet test {solution} -c Release --no-build {loggingFlags}");
.Name($"Test - {project}")
.Run($"dotnet test {project} -c Release --no-build {loggingFlags}");

job.Step()
.Name("Test report")
var id = $"test-report-{project.Replace("/", "-").Replace(".", "-")}";
job.Step(id)
.Name($"Test report - {project}")
.WorkingDirectory("test")
.Uses("dorny/test-reporter@31a54ee7ebcacc03a09ea97a7e5465a47b84aea5") // v1.9.1
.If("github.event == 'push' && (success() || failure())")
.If("github.event_name == 'push' && (success() || failure())")
.With(
("name", "Test Report"),
("path", "**/Tests.trx"),
("name", $"Test Report - {project}"),
("path", $"**/{logFileName}"),
("reporter", "dotnet-trx"),
("fail-on-error", "true"),
("fail-on-empty", "true"));

job.Step()
.Name("Publish test report link")
.Run($"echo \"[Test Results - {project}](${{{{ steps.{id}.outputs.url_html }}}})\" >> $GITHUB_STEP_SUMMARY");
}

public static Step StepPushToNuget(this Job job, bool pushAlways = false)
Expand Down Expand Up @@ -232,7 +244,8 @@ public static void StepInitializeCodeQl(this Job job) =>
.Uses("github/codeql-action/init@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0") // 3.28.9
.With(
("languages", "csharp"),
("build-mode", "manual"));
("build-mode", "manual"),
("db-location", "~/.codeql/databases"));

public static void StepPerformCodeQlAnalysis(this Job job) =>
job.Step()
Expand Down
Loading