Skip to content

Commit d693026

Browse files
TrinhChuongEETrinhChuongEE
authored
CDMS-769: adds additional metrics for display in the All services Grafana Dashboard. (#127)
* CDMS-769: adds additional metrics for display in the All services Grafana Dashboard. Removes logging of Decision and Error Notification which resulted in log being too large. * Code formatting * Add coverage and sonar fix * Code formatting
1 parent ab14311 commit d693026

File tree

9 files changed

+160
-21
lines changed

9 files changed

+160
-21
lines changed

‎BtmsGateway.Test/Services/Metrics/RequestMetricsTests.cs‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,70 @@ public void When_message_successfully_sent_Then_counter_is_incremented()
6969
sentMeasurements[1].ContainsTags(MetricsConstants.RequestTags.RouteAction).Should().BeTrue();
7070
sentMeasurements[1].Tags[MetricsConstants.RequestTags.RouteAction].Should().Be("Routing2");
7171
}
72+
73+
[Fact]
74+
public void When_request_completed_Then_requests_counter_is_incremented_and_request_duration_metric_is_emitted()
75+
{
76+
var metrics = ServiceProvider.GetRequiredService<IRequestMetrics>();
77+
var requestReceivedCollector = GetCollector<long>(MetricsConstants.InstrumentNames.RequestReceived);
78+
var requestDurationCollector = GetCollector<double>(MetricsConstants.InstrumentNames.RequestDuration);
79+
80+
metrics.RequestCompleted("/test-request-path-1", "POST", 204, 100);
81+
metrics.RequestCompleted("/test-request-path-2", "POST", 204, 200);
82+
83+
var receivedMeasurements = requestReceivedCollector.GetMeasurementSnapshot();
84+
receivedMeasurements.Count.Should().Be(2);
85+
receivedMeasurements[0].Value.Should().Be(1);
86+
receivedMeasurements[0].ContainsTags(MetricsConstants.RequestTags.RequestPath).Should().BeTrue();
87+
receivedMeasurements[0].Tags[MetricsConstants.RequestTags.RequestPath].Should().Be("/test-request-path-1");
88+
receivedMeasurements[0].ContainsTags(MetricsConstants.RequestTags.HttpMethod).Should().BeTrue();
89+
receivedMeasurements[0].Tags[MetricsConstants.RequestTags.HttpMethod].Should().Be("POST");
90+
receivedMeasurements[0].ContainsTags(MetricsConstants.RequestTags.StatusCode).Should().BeTrue();
91+
receivedMeasurements[0].Tags[MetricsConstants.RequestTags.StatusCode].Should().Be(204);
92+
receivedMeasurements[1].Value.Should().Be(1);
93+
receivedMeasurements[1].ContainsTags(MetricsConstants.RequestTags.RequestPath).Should().BeTrue();
94+
receivedMeasurements[1].Tags[MetricsConstants.RequestTags.RequestPath].Should().Be("/test-request-path-2");
95+
receivedMeasurements[1].ContainsTags(MetricsConstants.RequestTags.HttpMethod).Should().BeTrue();
96+
receivedMeasurements[1].Tags[MetricsConstants.RequestTags.HttpMethod].Should().Be("POST");
97+
receivedMeasurements[1].ContainsTags(MetricsConstants.RequestTags.StatusCode).Should().BeTrue();
98+
receivedMeasurements[1].Tags[MetricsConstants.RequestTags.StatusCode].Should().Be(204);
99+
100+
var durationMeasurements = requestDurationCollector.GetMeasurementSnapshot();
101+
durationMeasurements.Count.Should().Be(2);
102+
durationMeasurements[0].Value.Should().Be(100);
103+
durationMeasurements[0].ContainsTags(MetricsConstants.RequestTags.RequestPath).Should().BeTrue();
104+
durationMeasurements[0].Tags[MetricsConstants.RequestTags.RequestPath].Should().Be("/test-request-path-1");
105+
durationMeasurements[0].ContainsTags(MetricsConstants.RequestTags.HttpMethod).Should().BeTrue();
106+
durationMeasurements[0].Tags[MetricsConstants.RequestTags.HttpMethod].Should().Be("POST");
107+
durationMeasurements[0].ContainsTags(MetricsConstants.RequestTags.StatusCode).Should().BeTrue();
108+
durationMeasurements[0].Tags[MetricsConstants.RequestTags.StatusCode].Should().Be(204);
109+
durationMeasurements[1].Value.Should().Be(200);
110+
durationMeasurements[1].ContainsTags(MetricsConstants.RequestTags.RequestPath).Should().BeTrue();
111+
durationMeasurements[1].Tags[MetricsConstants.RequestTags.RequestPath].Should().Be("/test-request-path-2");
112+
durationMeasurements[1].ContainsTags(MetricsConstants.RequestTags.HttpMethod).Should().BeTrue();
113+
durationMeasurements[1].Tags[MetricsConstants.RequestTags.HttpMethod].Should().Be("POST");
114+
durationMeasurements[1].ContainsTags(MetricsConstants.RequestTags.StatusCode).Should().BeTrue();
115+
durationMeasurements[1].Tags[MetricsConstants.RequestTags.StatusCode].Should().Be(204);
116+
}
117+
118+
[Fact]
119+
public void When_request_faulted_Then_faulted_metric_is_emitted()
120+
{
121+
var metrics = ServiceProvider.GetRequiredService<IRequestMetrics>();
122+
var faultedCollector = GetCollector<long>(MetricsConstants.InstrumentNames.RequestFaulted);
123+
124+
metrics.RequestFaulted("/test-request-path-1", "POST", 500, new Exception("Test"));
125+
126+
var faultedMeasurements = faultedCollector.GetMeasurementSnapshot();
127+
faultedMeasurements.Count.Should().Be(1);
128+
faultedMeasurements[0].Value.Should().Be(1);
129+
faultedMeasurements[0].ContainsTags(MetricsConstants.RequestTags.RequestPath).Should().BeTrue();
130+
faultedMeasurements[0].Tags[MetricsConstants.RequestTags.RequestPath].Should().Be("/test-request-path-1");
131+
faultedMeasurements[0].ContainsTags(MetricsConstants.RequestTags.HttpMethod).Should().BeTrue();
132+
faultedMeasurements[0].Tags[MetricsConstants.RequestTags.HttpMethod].Should().Be("POST");
133+
faultedMeasurements[0].ContainsTags(MetricsConstants.RequestTags.StatusCode).Should().BeTrue();
134+
faultedMeasurements[0].Tags[MetricsConstants.RequestTags.StatusCode].Should().Be(500);
135+
faultedMeasurements[0].ContainsTags(MetricsConstants.RequestTags.ExceptionType).Should().BeTrue();
136+
faultedMeasurements[0].Tags[MetricsConstants.RequestTags.ExceptionType].Should().Be("Exception");
137+
}
72138
}

‎BtmsGateway.Test/TestUtils/TestWebServer.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ private TestWebServer(params ServiceDescriptor[] testServices)
7373

7474
var app = builder.Build();
7575

76+
app.UseMiddleware<MetricsMiddleware>();
7677
app.UseMiddleware<RoutingInterceptor>();
7778

7879
app.MapHealthChecks("/health");

‎BtmsGateway/Extensions/ServiceCollectionExtensions.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System.Diagnostics.CodeAnalysis;
22
using BtmsGateway.Config;
3+
using BtmsGateway.Middleware;
34
using BtmsGateway.Services.Metrics;
45
using BtmsGateway.Utils.Logging;
56
using Defra.TradeImportsDataApi.Api.Client;
@@ -71,6 +72,7 @@ public static IServiceCollection AddOperationalMetrics(this IServiceCollection s
7172
{
7273
services.AddSingleton<IRequestMetrics, RequestMetrics>();
7374
services.AddSingleton<IHealthMetrics, HealthMetrics>();
75+
services.AddTransient<MetricsMiddleware>();
7476

7577
return services;
7678
}

‎BtmsGateway/Middleware/MetricsMiddleware.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using System.Diagnostics.CodeAnalysis;
2+
using BtmsGateway.Services.Metrics;
3+
4+
namespace BtmsGateway.Middleware;
5+
6+
[ExcludeFromCodeCoverage]
7+
public class MetricsMiddleware(IRequestMetrics requestMetrics) : IMiddleware
8+
{
9+
public async Task InvokeAsync(HttpContext context, RequestDelegate next)
10+
{
11+
var startingTimestamp = TimeProvider.System.GetTimestamp();
12+
var path = (context.GetEndpoint() as RouteEndpoint)?.RoutePattern.RawText ?? context.Request.Path;
13+
try
14+
{
15+
await next(context);
16+
}
17+
catch (Exception ex)
18+
{
19+
requestMetrics.RequestFaulted(path, context.Request.Method, context.Response.StatusCode, ex);
20+
}
21+
finally
22+
{
23+
requestMetrics.RequestCompleted(
24+
path,
25+
context.Request.Method,
26+
context.Response.StatusCode,
27+
TimeProvider.System.GetElapsedTime(startingTimestamp).TotalMilliseconds
28+
);
29+
}
30+
}
31+
}

‎BtmsGateway/Program.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ static WebApplication BuildWebApplication(WebApplicationBuilder builder)
5656

5757
app.UseEmfExporter();
5858
app.UseHttpLogging();
59+
// Order of middleware matters!
60+
app.UseMiddleware<MetricsMiddleware>();
5961
app.UseMiddleware<RoutingInterceptor>();
6062
app.UseCustomHealthChecks();
6163
app.UseCheckRoutesEndpoints();

‎BtmsGateway/Services/Metrics/MetricsConstants.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ public static class RequestTags
1414
public const string RequestPath = "RequestPath";
1515
public const string Legend = "Legend";
1616
public const string RouteAction = "RouteAction";
17+
public const string HttpMethod = "HttpMethod";
18+
public const string StatusCode = "StatusCode";
19+
public const string ExceptionType = "ExceptionType";
1720
}
1821

1922
public static class ConsumerTags
@@ -37,6 +40,9 @@ public static class InstrumentNames
3740
{
3841
public const string MessagesReceived = "MessagesReceived";
3942
public const string MessagesSuccessfullySent = "MessagesSuccessfullySent";
43+
public const string RequestReceived = "RequestReceived";
44+
public const string RequestDuration = "RequestDuration";
45+
public const string RequestFaulted = "RequestFaulted";
4046
public const string MessagingConsume = "MessagingConsume";
4147
public const string MessagingConsumeErrors = "MessagingConsumeErrors";
4248
public const string MessagingConsumeActive = "MessagingConsumeActive";

‎BtmsGateway/Services/Metrics/RequestMetrics.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,17 @@ public interface IRequestMetrics
88
{
99
void MessageReceived(string? messageType, string? requestPath, string? legend, string routeAction);
1010
void MessageSuccessfullySent(string? messageType, string? requestPath, string? legend, string routeAction);
11+
void RequestCompleted(string requestPath, string httpMethod, int statusCode, double milliseconds);
12+
void RequestFaulted(string requestPath, string httpMethod, int statusCode, Exception exception);
1113
}
1214

1315
public class RequestMetrics : IRequestMetrics
1416
{
1517
private readonly Counter<long> messagesReceived;
1618
private readonly Counter<long> messagesSuccessfullySent;
19+
private readonly Counter<long> requestsReceived;
20+
private readonly Counter<long> requestsFaulted;
21+
private readonly Histogram<double> requestDuration;
1722

1823
public RequestMetrics(IMeterFactory meterFactory)
1924
{
@@ -30,6 +35,24 @@ public RequestMetrics(IMeterFactory meterFactory)
3035
Unit.COUNT.ToString(),
3136
"Count of messages successfully sent"
3237
);
38+
39+
requestsReceived = meter.CreateCounter<long>(
40+
MetricsConstants.InstrumentNames.RequestReceived,
41+
Unit.COUNT.ToString(),
42+
"Count of messages received"
43+
);
44+
45+
requestDuration = meter.CreateHistogram<double>(
46+
MetricsConstants.InstrumentNames.RequestDuration,
47+
Unit.MILLISECONDS.ToString(),
48+
"Duration of request"
49+
);
50+
51+
requestsFaulted = meter.CreateCounter<long>(
52+
MetricsConstants.InstrumentNames.RequestFaulted,
53+
Unit.COUNT.ToString(),
54+
"Count of request faults"
55+
);
3356
}
3457

3558
public void MessageReceived(string? messageType, string? requestPath, string? legend, string routeAction)
@@ -42,6 +65,19 @@ public void MessageSuccessfullySent(string? messageType, string? requestPath, st
4265
messagesSuccessfullySent.Add(1, BuildTags(messageType, requestPath, legend, routeAction));
4366
}
4467

68+
public void RequestCompleted(string requestPath, string httpMethod, int statusCode, double milliseconds)
69+
{
70+
requestsReceived.Add(1, BuildRequestTags(requestPath, httpMethod, statusCode));
71+
requestDuration.Record(milliseconds, BuildRequestTags(requestPath, httpMethod, statusCode));
72+
}
73+
74+
public void RequestFaulted(string requestPath, string httpMethod, int statusCode, Exception exception)
75+
{
76+
var tagList = BuildRequestTags(requestPath, httpMethod, statusCode);
77+
tagList.Add(MetricsConstants.RequestTags.ExceptionType, exception.GetType().Name);
78+
requestsFaulted.Add(1, tagList);
79+
}
80+
4581
private static TagList BuildTags(string? messageType, string? requestPath, string? legend, string routeAction)
4682
{
4783
return new TagList
@@ -53,4 +89,15 @@ private static TagList BuildTags(string? messageType, string? requestPath, strin
5389
{ MetricsConstants.RequestTags.RouteAction, routeAction },
5490
};
5591
}
92+
93+
private static TagList BuildRequestTags(string requestPath, string httpMethod, int statusCode)
94+
{
95+
return new TagList
96+
{
97+
{ MetricsConstants.RequestTags.Service, Process.GetCurrentProcess().ProcessName },
98+
{ MetricsConstants.RequestTags.RequestPath, requestPath },
99+
{ MetricsConstants.RequestTags.HttpMethod, httpMethod },
100+
{ MetricsConstants.RequestTags.StatusCode, statusCode },
101+
};
102+
}
56103
}

‎BtmsGateway/Services/Routing/DecisionSender.cs‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -145,19 +145,11 @@ await _featureManager.IsEnabledAsync(Features.SendOnlyBtmsDecisionToCds)
145145

146146
if (string.IsNullOrWhiteSpace(comparerDecision))
147147
{
148-
_logger.Error(
149-
"{MRN} Decision Comparer returned an invalid decision: {ComparerDecision}.",
150-
mrn,
151-
comparerDecision
152-
);
148+
_logger.Error("{MRN} Decision Comparer returned an invalid decision", mrn);
153149
throw new DecisionComparisonException($"{mrn} Decision Comparer returned an invalid decision.");
154150
}
155151

156-
_logger.Information(
157-
"{MRN} Received Decision from Decision Comparer: {ComparerDecision}",
158-
mrn,
159-
comparerDecision
160-
);
152+
_logger.Information("{MRN} Received Decision from Decision Comparer to send to CDS", mrn);
161153
// Just log decision for now. Eventually, in cut over, will send the Decision to CDS.
162154
// Ensure original ALVS request headers are passed through and appended in SendCdsFormattedSoapMessageAsync!
163155
// Pass the CDS response back out!

‎BtmsGateway/Services/Routing/ErrorNotificationSender.cs‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ public async Task<RoutingResult> SendErrorNotificationAsync(
8383
throw new DecisionComparisonException($"{mrn} Failed to send Error Notification to Decision Comparer.");
8484
}
8585

86-
ForwardErrorNotificationAsync(mrn, messageSource, errorNotification);
86+
ForwardErrorNotificationAsync(mrn, messageSource);
8787

8888
return routingResult with
8989
{
@@ -98,19 +98,11 @@ public async Task<RoutingResult> SendErrorNotificationAsync(
9898
};
9999
}
100100

101-
private void ForwardErrorNotificationAsync(
102-
string? mrn,
103-
MessagingConstants.MessageSource messageSource,
104-
string errorNotification
105-
)
101+
private void ForwardErrorNotificationAsync(string? mrn, MessagingConstants.MessageSource messageSource)
106102
{
107103
if (messageSource == MessagingConstants.MessageSource.Btms)
108104
{
109-
_logger.Information(
110-
"{MRN} Produced Error Notification to send to CDS: {ErrorNotification}",
111-
mrn,
112-
errorNotification
113-
);
105+
_logger.Information("{MRN} Produced Error Notification to send to CDS", mrn);
114106
// Just log error notification for now. Eventually, in cut over, will send the notification to CDS.
115107
// Ensure original ALVS request headers are passed through and appended in the CDS request!
116108
// Pass the CDS response back out!

0 commit comments

Comments
 (0)