-
Notifications
You must be signed in to change notification settings - Fork 351
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merged PR 875: add readiness/liveness probes to workflow
add healthz file creation based on health check status now workflow adds a readiness/liveness health check, and also a health check publisher. This way, the health check system periodically executes this check. - create file when reporting healthy - delete file when reporting unhealthy - add health check - add health check publisher to o the service container, - add publisher unit tests - add readiness/liveness probes cfg solved: #133010
- Loading branch information
1 parent
78cd88b
commit 9c5dbca
Showing
6 changed files
with
288 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
120 changes: 120 additions & 0 deletions
120
src/shipping/workflow/Fabrikam.Workflow.Service.Tests/ReadinessLivenessPublisherTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| // ------------------------------------------------------------ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. | ||
| // ------------------------------------------------------------ | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Diagnostics.HealthChecks; | ||
| using Microsoft.Extensions.Logging; | ||
| using Fabrikam.Workflow.Service.Services; | ||
| using Moq; | ||
| using Xunit; | ||
|
|
||
| namespace Fabrikam.Workflow.Service.Tests | ||
| { | ||
| public class ReadinessLivenessPublisherTests | ||
| { | ||
| private const int DelayCompletionMs = 1000; | ||
|
|
||
| private readonly ReadinessLivenessPublisher _publisher; | ||
|
|
||
| public ReadinessLivenessPublisherTests() | ||
| { | ||
| var servicesBuilder = new ServiceCollection(); | ||
| servicesBuilder.AddLogging(logging => logging.AddDebug()); | ||
| var services = servicesBuilder.BuildServiceProvider(); | ||
|
|
||
| _publisher = | ||
| new ReadinessLivenessPublisher( | ||
| services.GetService<ILogger<ReadinessLivenessPublisher>>()); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task WhenPublishingAndReportIsHealthy_FileExists() | ||
| { | ||
| // Arrange | ||
| var healthReportEntries = new Dictionary<string, HealthReportEntry>() | ||
| { | ||
| {"healthy", new HealthReportEntry(HealthStatus.Healthy, null,TimeSpan.MinValue, null, null) } | ||
| }; | ||
|
|
||
| // Act | ||
| await _publisher.PublishAsync( | ||
| new HealthReport(healthReportEntries, TimeSpan.MinValue), | ||
| new CancellationTokenSource().Token); | ||
|
|
||
| // Arrange | ||
| Assert.True(File.Exists(ReadinessLivenessPublisher.FilePath)); | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task WhenPublishingAndReportIsUnhealthy_FileDateTimeIsNotModified() | ||
| { | ||
| // Arrange | ||
| var healthReportEntries = new Dictionary<string, HealthReportEntry>() | ||
| { | ||
| {"healthy", new HealthReportEntry(HealthStatus.Healthy, null,TimeSpan.MinValue, null, null) } | ||
| }; | ||
|
|
||
| await _publisher.PublishAsync( | ||
| new HealthReport( | ||
| healthReportEntries, | ||
| TimeSpan.MinValue), | ||
| new CancellationTokenSource().Token); | ||
|
|
||
| healthReportEntries.Add( | ||
| "unhealthy", | ||
| new HealthReportEntry( | ||
| HealthStatus.Unhealthy, | ||
| null,TimeSpan.MinValue, null, null)); | ||
|
|
||
| // Act | ||
| DateTime healthyWriteTime = File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath); | ||
| await _publisher.PublishAsync( | ||
| new HealthReport(healthReportEntries, TimeSpan.MinValue), | ||
| new CancellationTokenSource().Token); | ||
|
|
||
| // Arrange | ||
| Assert.True(File.Exists(ReadinessLivenessPublisher.FilePath)); | ||
| Assert.Equal(healthyWriteTime, File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath)); | ||
| } | ||
|
|
||
| [Fact(Timeout = DelayCompletionMs * 3)] | ||
| public async Task WhenPublishingAndReportIsHealthyTwice_FileDateTimeIsModified() | ||
| { | ||
| // Arrange | ||
| Func<Task> emulatePeriodicHealthCheckAsync = | ||
| () => Task.Delay(DelayCompletionMs); | ||
|
|
||
| var healthReportEntries = new Dictionary<string, HealthReportEntry>() | ||
| { | ||
| {"healthy", new HealthReportEntry(HealthStatus.Healthy, null,TimeSpan.MinValue, null, null) } | ||
| }; | ||
|
|
||
| // Act | ||
| await _publisher.PublishAsync( | ||
| new HealthReport( | ||
| healthReportEntries, | ||
| TimeSpan.MinValue), | ||
| new CancellationTokenSource().Token); | ||
|
|
||
| DateTime firstTimehealthyWriteTime = File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath); | ||
|
|
||
| await emulatePeriodicHealthCheckAsync(); | ||
|
|
||
| await _publisher.PublishAsync( | ||
| new HealthReport(healthReportEntries, TimeSpan.MinValue), | ||
| new CancellationTokenSource().Token); | ||
|
|
||
| DateTime sencondTimehealthyWriteTime = File.GetLastWriteTime(ReadinessLivenessPublisher.FilePath); | ||
|
|
||
| // Arrange | ||
| Assert.True(firstTimehealthyWriteTime < sencondTimehealthyWriteTime); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
81 changes: 81 additions & 0 deletions
81
src/shipping/workflow/Fabrikam.Workflow.Service/Services/ReadinessLivenessPublisher.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| // ------------------------------------------------------------ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License (MIT). See License.txt in the repo root for license information. | ||
| // ------------------------------------------------------------ | ||
|
|
||
| using System; | ||
| using System.IO; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Extensions.Diagnostics.HealthChecks; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| namespace Fabrikam.Workflow.Service.Services | ||
| { | ||
| public class ReadinessLivenessPublisher : IHealthCheckPublisher | ||
| { | ||
| public const string FilePath = "healthz"; | ||
|
|
||
| private readonly ILogger _logger; | ||
|
|
||
| public ReadinessLivenessPublisher(ILogger<ReadinessLivenessPublisher> logger) | ||
| { | ||
| this._logger = logger; | ||
| } | ||
|
|
||
| public Task PublishAsync(HealthReport report, | ||
| CancellationToken cancellationToken) | ||
| { | ||
| switch (report.Status) | ||
| { | ||
| case HealthStatus.Healthy: | ||
| { | ||
| this._logger.LogInformation( | ||
| "{Timestamp} Readiness/Liveness Probe Status: {Result}", | ||
| DateTime.UtcNow, | ||
| report.Status); | ||
|
|
||
| CreateOrUpdateHealthz(); | ||
|
|
||
| break; | ||
| } | ||
|
|
||
| case HealthStatus.Degraded: | ||
| { | ||
| this._logger.LogWarning( | ||
| "{Timestamp} Readiness/Liveness Probe Status: {Result}", | ||
| DateTime.UtcNow, | ||
| report.Status); | ||
|
|
||
| break; | ||
| } | ||
|
|
||
| case HealthStatus.Unhealthy: | ||
| { | ||
| this._logger.LogError( | ||
| "{Timestamp} Readiness Probe/Liveness Status: {Result}", | ||
| DateTime.UtcNow, | ||
| report.Status); | ||
|
|
||
| break; | ||
| } | ||
| } | ||
|
|
||
| cancellationToken.ThrowIfCancellationRequested(); | ||
|
|
||
| return Task.CompletedTask; | ||
| } | ||
|
|
||
| private static void CreateOrUpdateHealthz() | ||
| { | ||
| if (File.Exists(FilePath)) | ||
| { | ||
| File.SetLastWriteTimeUtc(FilePath, DateTime.UtcNow); | ||
| } | ||
| else | ||
| { | ||
| File.AppendText(FilePath).Close(); | ||
| } | ||
| } | ||
| } | ||
| } |