Skip to content

Commit 340e260

Browse files
jvydenjvyden
authored
Add dotnet-counters compatible metrics to GameServer (#848)
I've been experimenting with dotnet counters and Prometheus in production, and these feel like good additions to the metrics I've already been recording through `System.Runtime`. - Api/Game requests served, timings, and errors - Image conversion totals and timings - Track dry/remote DataStore checks, served assets, and timings To produce request timings I've refactored `RequestStatisticTrackingService` to be a middleware instead, and added Stopwatches to our custom DataStores and our image import process.
2 parents 9f1fb07 + b3184d0 commit 340e260

9 files changed

Lines changed: 172 additions & 40 deletions

File tree

‎Refresh.Core/Importing/ImageImporter.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
using System.Collections.Concurrent;
2+
using System.Diagnostics;
23
using Bunkum.Core.Storage;
34
using NotEnoughLogs;
45
using Refresh.Common.Helpers;
6+
using Refresh.Core.Metrics;
57
using Refresh.Database;
68
using Refresh.Database.Models.Assets;
79
using Refresh.Database.Models.Authentication;
@@ -92,10 +94,13 @@ public void ImportAsset(string hash, bool isPsp, GameAssetType? type, IDataStore
9294
using Stream stream = dataStore.GetStreamFromStore(dataStorePath);
9395
using Stream writeStream = dataStore.OpenWriteStream($"png/{hash}");
9496

97+
Stopwatch sw = Stopwatch.StartNew();
98+
9599
switch (type)
96100
{
97101
case GameAssetType.GameDataTexture:
98102
GtfToPng(stream, writeStream);
103+
GameServerMetrics.RecordImageConversion(sw);
99104
break;
100105
case GameAssetType.Mip: {
101106
byte[] rawData = dataStore.GetDataFromStore(dataStorePath);
@@ -104,14 +109,17 @@ public void ImportAsset(string hash, bool isPsp, GameAssetType? type, IDataStore
104109
using MemoryStream dataStream = new(data);
105110

106111
MipToPng(dataStream, writeStream);
112+
GameServerMetrics.RecordImageConversion(sw);
107113
break;
108114
}
109115
case GameAssetType.Texture:
110116
TextureToPng(stream, writeStream);
117+
GameServerMetrics.RecordImageConversion(sw);
111118
break;
112119
case GameAssetType.Tga:
113120
case GameAssetType.Jpeg:
114121
ImageToPng(stream, writeStream);
122+
GameServerMetrics.RecordImageConversion(sw);
115123
break;
116124
case GameAssetType.Png:
117125
stream.CopyTo(writeStream); // TODO: use hard links instead of just replicating same data, or run 'optipng'?

‎Refresh.Core/Metrics/DataStoreMetrics.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
using System.Diagnostics;
2+
using System.Diagnostics.Metrics;
3+
4+
namespace Refresh.Core.Metrics;
5+
6+
/// <summary>
7+
/// Metrics regarding timings and totals of DataStores.
8+
/// </summary>
9+
public static class DataStoreMetrics
10+
{
11+
private static readonly Meter Meter = new("refresh.core.storage");
12+
13+
private static readonly Counter<int> AssetsServed = Meter.CreateCounter<int>("refresh.core.storage.served");
14+
private static readonly Histogram<double> AssetsTime = Meter.CreateHistogram<double>("refresh.core.storage.timing");
15+
16+
private static readonly Counter<int> RemoteDataStoreChecks = Meter.CreateCounter<int>("refresh.core.storage.remote.checks");
17+
18+
private static void Record(Stopwatch sw, string type)
19+
{
20+
KeyValuePair<string, object?> kvp = new("dataStore", type);
21+
AssetsServed.Add(1, kvp);
22+
AssetsTime.Record(sw.Elapsed.TotalMilliseconds, kvp);
23+
sw.Stop();
24+
}
25+
26+
public static void RecordDry(Stopwatch sw) => Record(sw, "dry");
27+
public static void RecordRemote(Stopwatch sw) => Record(sw, "remote");
28+
public static void RecordRemoteCheck() => RemoteDataStoreChecks.Add(1);
29+
}

‎Refresh.Core/Metrics/GameServerMetrics.cs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
using System.Diagnostics;
2+
using System.Diagnostics.Metrics;
3+
using System.Net;
4+
5+
namespace Refresh.Core.Metrics;
6+
7+
/// <summary>
8+
/// Metrics regarding timings and totals of certain events taking place during this instance of the game server.
9+
/// </summary>
10+
/// <remarks>
11+
/// These are different from metrics like RequestStatisticTrackingService because those are stored totals in the database.
12+
/// These are instead reported as dotnet metrics, which can be consumed by Prometheus and picked apart per-instance per-lifetime per-pod.
13+
/// </remarks>
14+
public static class GameServerMetrics
15+
{
16+
private static readonly Meter Meter = new("refresh.gameserver");
17+
18+
private static readonly Counter<int> ApiRequestsServed = Meter.CreateCounter<int>("refresh.gameserver.requests_served.api");
19+
private static readonly Histogram<double> ApiRequestTime = Meter.CreateHistogram<double>("refresh.gameserver.request_timing.api");
20+
private static readonly Counter<int> ApiRequestErrors = Meter.CreateCounter<int>("refresh.gameserver.request_errors.api");
21+
22+
private static readonly Counter<int> GameRequestsServed = Meter.CreateCounter<int>("refresh.gameserver.requests_served.game");
23+
private static readonly Histogram<double> GameRequestTime = Meter.CreateHistogram<double>("refresh.gameserver.request_timing.game");
24+
private static readonly Counter<int> GameRequestErrors = Meter.CreateCounter<int>("refresh.gameserver.request_errors.api");
25+
26+
private static readonly Counter<int> ImageConversions = Meter.CreateCounter<int>("refresh.gameserver.images_converted");
27+
private static readonly Histogram<double> ImageConversionTime = Meter.CreateHistogram<double>("refresh.gameserver.image_conversion_timing");
28+
29+
private static bool IsError(HttpStatusCode code)
30+
{
31+
return (int)code >= 400;
32+
}
33+
34+
public static void RecordApiRequest(Stopwatch sw, HttpStatusCode code)
35+
{
36+
ApiRequestsServed.Add(1);
37+
ApiRequestTime.Record(sw.Elapsed.TotalMilliseconds);
38+
39+
if(IsError(code))
40+
ApiRequestErrors.Add(1, new KeyValuePair<string, object?>("code", code));
41+
42+
sw.Stop();
43+
}
44+
45+
public static void RecordGameRequest(Stopwatch sw, HttpStatusCode code)
46+
{
47+
GameRequestsServed.Add(1);
48+
GameRequestTime.Record(sw.Elapsed.TotalMilliseconds);
49+
50+
if(IsError(code))
51+
GameRequestErrors.Add(1, new KeyValuePair<string, object?>("code", code));
52+
53+
sw.Stop();
54+
}
55+
56+
public static void RecordImageConversion(Stopwatch sw)
57+
{
58+
ImageConversions.Add(1);
59+
ImageConversionTime.Record(sw.Elapsed.TotalMilliseconds);
60+
sw.Stop();
61+
}
62+
}

‎Refresh.Core/Metrics/RequestStatisticTrackingMiddleware.cs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
using System.Diagnostics;
2+
using Bunkum.Core.Database;
3+
using Bunkum.Core.Endpoints.Middlewares;
4+
using Bunkum.Listener.Request;
5+
6+
namespace Refresh.Core.Metrics;
7+
8+
public class RequestStatisticTrackingMiddleware : IMiddleware
9+
{
10+
private static int _gameRequestsToSubmit;
11+
private static int _apiRequestsToSubmit;
12+
13+
public static (int game, int api) SubmitAndClearRequests()
14+
{
15+
return (game: Interlocked.Exchange(ref _gameRequestsToSubmit, 0),
16+
api: Interlocked.Exchange(ref _apiRequestsToSubmit, 0));
17+
}
18+
19+
private readonly ThreadLocal<Stopwatch> _sw = new(() => new Stopwatch());
20+
21+
public void HandleRequest(ListenerContext context, Lazy<IDatabaseContext> database, Action next)
22+
{
23+
Debug.Assert(this._sw.Value != null);
24+
this._sw.Value.Restart();
25+
next();
26+
this._sw.Value.Stop();
27+
28+
if (context.Uri.AbsolutePath.StartsWith("/lbp/" /* GameEndpointAttribute.BaseRoute */))
29+
{
30+
Interlocked.Increment(ref _gameRequestsToSubmit);
31+
GameServerMetrics.RecordGameRequest(this._sw.Value, context.ResponseCode);
32+
}
33+
else
34+
{
35+
Interlocked.Increment(ref _apiRequestsToSubmit);
36+
GameServerMetrics.RecordApiRequest(this._sw.Value, context.ResponseCode);
37+
}
38+
}
39+
}

‎Refresh.Core/Storage/DryDataStore.cs‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1+
using System.Diagnostics;
12
using System.Text;
23
using Bunkum.Core.Storage;
34
using JetBrains.Annotations;
45
using Refresh.Common.Verification;
6+
using Refresh.Core.Metrics;
57

68
namespace Refresh.Core.Storage;
79

@@ -16,6 +18,8 @@ public DryDataStore(DryArchiveConfig config)
1618
this._config = config;
1719
}
1820

21+
private readonly ThreadLocal<Stopwatch> _sw = new(() => new Stopwatch());
22+
1923
[Pure]
2024
private string? GetPath(ReadOnlySpan<char> hash)
2125
{
@@ -67,7 +71,12 @@ public byte[] GetDataFromStore(string key)
6771
if (path == null)
6872
throw new FormatException("The key was invalid.");
6973

70-
return File.ReadAllBytes(path);
74+
this._sw.Value!.Restart();
75+
byte[] data = File.ReadAllBytes(path);
76+
this._sw.Value!.Stop();
77+
DataStoreMetrics.RecordDry(this._sw.Value);
78+
79+
return data;
7180
}
7281

7382
public bool RemoveFromStore(string key)
@@ -89,7 +98,12 @@ public Stream GetStreamFromStore(string key)
8998
if (path == null)
9099
throw new FormatException("The key was invalid.");
91100

92-
return File.OpenRead(path);
101+
this._sw.Value!.Restart();
102+
Stream data = File.OpenRead(path);
103+
this._sw.Value!.Stop();
104+
DataStoreMetrics.RecordDry(this._sw.Value);
105+
106+
return data;
93107
}
94108

95109
public Stream OpenWriteStream(string key)

‎Refresh.Core/Storage/RemoteRefreshDataStore.cs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1+
using System.Diagnostics;
12
using Bunkum.Core.Storage;
23
using JetBrains.Annotations;
4+
using Refresh.Core.Metrics;
5+
36
#pragma warning disable CS0162 // Unreachable code detected
47

58
namespace Refresh.Core.Storage;
@@ -12,13 +15,15 @@ public class RemoteRefreshDataStore : IDataStore
1215
{
1316
private const string SourceUrl = "https://lbp.lbpbonsai.com/api/v3/";
1417
private const string WriteError = $"{nameof(RemoteRefreshDataStore)} is a read-only source, and cannot be written to.";
15-
private const bool HideImages = false;
18+
private const bool HideImages = true;
1619

1720
private readonly HttpClient _client = new()
1821
{
1922
BaseAddress = new Uri(SourceUrl),
2023
};
2124

25+
private readonly ThreadLocal<Stopwatch> _sw = new(() => new Stopwatch());
26+
2227
[Pure]
2328
private string? GetPath(ReadOnlySpan<char> key)
2429
{
@@ -44,6 +49,7 @@ public bool ExistsInStore(string key)
4449
if (path == null)
4550
throw new FormatException("The key was invalid.");
4651

52+
DataStoreMetrics.RecordRemoteCheck();
4753
HttpResponseMessage resp = this.Get(path);
4854
return resp.IsSuccessStatusCode;
4955
}
@@ -56,11 +62,16 @@ public byte[] GetDataFromStore(string key)
5662

5763
if (path == null)
5864
throw new FormatException("The key was invalid.");
59-
65+
66+
this._sw.Value!.Restart();
6067
HttpResponseMessage resp = this.Get(path);
6168
resp.EnsureSuccessStatusCode();
6269

63-
return resp.Content.ReadAsByteArrayAsync().Result;
70+
byte[] data = resp.Content.ReadAsByteArrayAsync().Result;
71+
this._sw.Value!.Stop();
72+
DataStoreMetrics.RecordRemote(this._sw.Value);
73+
74+
return data;
6475
}
6576

6677
public bool RemoveFromStore(string key) => throw new InvalidOperationException(WriteError);

‎Refresh.GameServer/RefreshGameServer.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
using Refresh.Core.Configuration;
1616
using Refresh.Core.Extensions;
1717
using Refresh.Core.Importing;
18+
using Refresh.Core.Metrics;
1819
using Refresh.Core.Services;
1920
using Refresh.Core.Storage;
2021
using Refresh.Core.Types.Categories;
@@ -141,6 +142,7 @@ protected override void SetupMiddlewares()
141142
this.Server.AddMiddleware<CrossOriginMiddleware>();
142143
this.Server.AddMiddleware<PspVersionMiddleware>();
143144
this.Server.AddMiddleware(new PresenceAuthenticationMiddleware(this._integrationConfig!));
145+
this.Server.AddMiddleware<RequestStatisticTrackingMiddleware>();
144146
}
145147

146148
protected override void SetupConfiguration()
@@ -183,7 +185,6 @@ protected override void SetupServices()
183185

184186
this.Server.AddService<RoleService>();
185187
this.Server.AddService<SmtpService>();
186-
this.Server.AddService<RequestStatisticTrackingService>();
187188
this.Server.AddService<PresenceService>();
188189
this.Server.AddService<PlayNowService>();
189190
this.Server.AddService<CommandService>();

‎Refresh.Interfaces.Workers/RequestTracking/RequestStatisticSubmitWorker.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using Refresh.Core.Metrics;
12
using Refresh.Core.Types.Data;
23

34
namespace Refresh.Interfaces.Workers.RequestTracking;
@@ -8,7 +9,7 @@ public class RequestStatisticSubmitWorker : IWorker
89

910
public void DoWork(DataContext context)
1011
{
11-
(int game, int api) = RequestStatisticTrackingService.SubmitAndClearRequests();
12+
(int game, int api) = RequestStatisticTrackingMiddleware.SubmitAndClearRequests();
1213

1314
context.Database.IncrementRequests(api, game);
1415
}

‎Refresh.Interfaces.Workers/RequestTracking/RequestStatisticTrackingService.cs‎

Lines changed: 0 additions & 33 deletions
This file was deleted.

0 commit comments

Comments
 (0)