Skip to content

Commit 8cf3f88

Browse files
authored
Merge pull request #142 from serilog-contrib/dev
prevent duplicate rowkey
2 parents e5947e2 + 34a3e96 commit 8cf3f88

File tree

9 files changed

+243
-13
lines changed

9 files changed

+243
-13
lines changed

samples/SampleStressTest/Program.cs

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
using Azure.Data.Tables;
2+
3+
using Microsoft.Extensions.Configuration;
4+
using Microsoft.Extensions.DependencyInjection;
5+
using Microsoft.Extensions.DependencyInjection.Extensions;
6+
using Microsoft.Extensions.Hosting;
7+
8+
using Serilog;
9+
10+
namespace SampleStressTest;
11+
12+
internal static class Program
13+
{
14+
public static async Task<int> Main(string[] args)
15+
{
16+
try
17+
{
18+
var app = Host
19+
.CreateDefaultBuilder(args)
20+
.ConfigureServices(services => services
21+
.AddHostedService<StressTestService>()
22+
.TryAddSingleton(sp =>
23+
{
24+
var configuration = sp.GetRequiredService<IConfiguration>();
25+
var connectionString = configuration.GetConnectionString("StorageAccount");
26+
return new TableServiceClient(connectionString);
27+
})
28+
)
29+
.UseSerilog((context, services, configuration) => configuration
30+
.ReadFrom.Services(services)
31+
.Enrich.FromLogContext()
32+
.WriteTo.AzureTableStorage(
33+
storageAccount: services.GetRequiredService<TableServiceClient>(),
34+
storageTableName: "SampleStressTest"
35+
)
36+
)
37+
.Build();
38+
39+
await app.RunAsync();
40+
41+
return 0;
42+
}
43+
catch (Exception ex)
44+
{
45+
Log.Fatal(ex, "Host terminated unexpectedly");
46+
return 1;
47+
}
48+
finally
49+
{
50+
await Log.CloseAndFlushAsync();
51+
}
52+
}
53+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<None Remove="appsettings.json" />
12+
</ItemGroup>
13+
14+
<ItemGroup>
15+
<Content Include="appsettings.json">
16+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
17+
<ExcludeFromSingleFile>true</ExcludeFromSingleFile>
18+
<CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
19+
</Content>
20+
</ItemGroup>
21+
22+
<ItemGroup>
23+
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
24+
<PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
25+
</ItemGroup>
26+
27+
<ItemGroup>
28+
<ProjectReference Include="..\..\src\Serilog.Sinks.AzureTableStorage\Serilog.Sinks.AzureTableStorage.csproj" />
29+
</ItemGroup>
30+
31+
</Project>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
using Microsoft.Extensions.Hosting;
2+
using Microsoft.Extensions.Logging;
3+
4+
namespace SampleStressTest;
5+
6+
public class StressTestService : IHostedLifecycleService
7+
{
8+
private readonly ILogger<StressTestService> _logger;
9+
10+
public StressTestService(ILogger<StressTestService> logger)
11+
{
12+
this._logger = logger;
13+
}
14+
15+
public Task StartAsync(CancellationToken cancellationToken)
16+
{
17+
_logger.LogInformation("StartAsync Called");
18+
return Task.CompletedTask;
19+
}
20+
21+
public Task StartedAsync(CancellationToken cancellationToken)
22+
{
23+
_logger.LogInformation("StartedAsync Called");
24+
25+
// hammer the logger
26+
var result = Parallel.For(1, 10000, index =>
27+
{
28+
_logger.LogInformation("Logging Loop {index}", index);
29+
_logger.LogInformation("Another Entry {index}", index);
30+
_logger.LogInformation("Duplicate Entry {index}", index);
31+
});
32+
33+
_logger.LogInformation("Stress test: {completed}", result.IsCompleted);
34+
35+
return Task.CompletedTask;
36+
}
37+
38+
public Task StartingAsync(CancellationToken cancellationToken)
39+
{
40+
_logger.LogInformation("StartingAsync Called");
41+
return Task.CompletedTask;
42+
}
43+
44+
public Task StopAsync(CancellationToken cancellationToken)
45+
{
46+
_logger.LogInformation("StopAsync Called");
47+
return Task.CompletedTask;
48+
}
49+
50+
public Task StoppedAsync(CancellationToken cancellationToken)
51+
{
52+
_logger.LogInformation("StoppedAsync Called");
53+
return Task.CompletedTask;
54+
}
55+
56+
public Task StoppingAsync(CancellationToken cancellationToken)
57+
{
58+
_logger.LogInformation("StoppingAsync Called");
59+
return Task.CompletedTask;
60+
}
61+
62+
}
+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"Logging": {
3+
"LogLevel": {
4+
"Default": "Information",
5+
"Microsoft.AspNetCore": "Warning"
6+
}
7+
},
8+
"ConnectionStrings": {
9+
"StorageAccount": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
10+
},
11+
"AllowedHosts": "*"
12+
}

samples/SampleWebApplication/Pages/Logs.cshtml

+5-2
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@
6666
<colgroup>
6767
<col style="width: 30px" />
6868
<col style="width: 30px" />
69-
<col style="width: 200px" />
69+
<col style="width: 100px" />
70+
<col style="width: 150px" />
7071
<col style="width: 120px" />
7172
<col style="" />
7273
</colgroup>
@@ -75,6 +76,7 @@
7576
<th scope="col"></th>
7677
<th scope="col"></th>
7778
<th scope="col">Date</th>
79+
<th scope="col">Time</th>
7880
<th scope="col">Level</th>
7981
<th scope="col">Message</th>
8082
</tr>
@@ -111,7 +113,8 @@
111113
break;
112114
}
113115
</td>
114-
<td>@log.Timestamp?.ToString("g")</td>
116+
<td>@log.Timestamp?.ToString("d")</td>
117+
<td>@log.Timestamp?.ToString("hh:mm:ss.fffffff")</td>
115118
<td>@log.Level</td>
116119
<td>@log.RenderedMessage</td>
117120
</tr>

serilog-sinks-azuretablestorage.sln

+7
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SampleWebApplication", "sam
2020
EndProject
2121
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SampleConsoleApplication", "samples\SampleConsoleApplication\SampleConsoleApplication.csproj", "{2F82E3DC-7071-41AD-A1E9-9CB70CF27CDA}"
2222
EndProject
23+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SampleStressTest", "samples\SampleStressTest\SampleStressTest.csproj", "{31F5B01D-F6D1-4802-B412-167A424F3818}"
24+
EndProject
2325
Global
2426
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2527
Debug|Any CPU = Debug|Any CPU
@@ -42,13 +44,18 @@ Global
4244
{2F82E3DC-7071-41AD-A1E9-9CB70CF27CDA}.Debug|Any CPU.Build.0 = Debug|Any CPU
4345
{2F82E3DC-7071-41AD-A1E9-9CB70CF27CDA}.Release|Any CPU.ActiveCfg = Release|Any CPU
4446
{2F82E3DC-7071-41AD-A1E9-9CB70CF27CDA}.Release|Any CPU.Build.0 = Release|Any CPU
47+
{31F5B01D-F6D1-4802-B412-167A424F3818}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
48+
{31F5B01D-F6D1-4802-B412-167A424F3818}.Debug|Any CPU.Build.0 = Debug|Any CPU
49+
{31F5B01D-F6D1-4802-B412-167A424F3818}.Release|Any CPU.ActiveCfg = Release|Any CPU
50+
{31F5B01D-F6D1-4802-B412-167A424F3818}.Release|Any CPU.Build.0 = Release|Any CPU
4551
EndGlobalSection
4652
GlobalSection(SolutionProperties) = preSolution
4753
HideSolutionNode = FALSE
4854
EndGlobalSection
4955
GlobalSection(NestedProjects) = preSolution
5056
{420A3E41-C520-420D-95E6-24CF86BF7477} = {18BA35F3-24C4-4297-859A-C3DCB2D00488}
5157
{2F82E3DC-7071-41AD-A1E9-9CB70CF27CDA} = {18BA35F3-24C4-4297-859A-C3DCB2D00488}
58+
{31F5B01D-F6D1-4802-B412-167A424F3818} = {18BA35F3-24C4-4297-859A-C3DCB2D00488}
5259
EndGlobalSection
5360
GlobalSection(ExtensibilityGlobals) = postSolution
5461
SolutionGuid = {73E16F7E-6DF4-4355-BF3C-73DDBBA1867E}

src/Serilog.Sinks.AzureTableStorage/LoggerConfigurationAzureTableStorageExtensions.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public static class LoggerConfigurationAzureTableStorageExtensions
3838
/// A reasonable default for the number of events posted in
3939
/// each batch.
4040
/// </summary>
41-
public const int DefaultBatchSizeLimit = 100;
41+
public const int DefaultBatchSizeLimit = 1000;
4242

4343
/// <summary>
4444
/// A reasonable default time to wait between checking for event batches.

src/Serilog.Sinks.AzureTableStorage/Sinks/AzureTableStorage/DefaultKeyGenerator.cs

+68-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Threading;
23

34
using Serilog.Events;
45
using Serilog.Sinks.AzureTableStorage.Extensions;
@@ -10,6 +11,7 @@ namespace Serilog.Sinks.AzureTableStorage;
1011
/// </summary>
1112
public class DefaultKeyGenerator : IKeyGenerator
1213
{
14+
1315
/// <summary>
1416
/// Automatically generates the PartitionKey based on the logEvent timestamp
1517
/// </summary>
@@ -33,7 +35,7 @@ public virtual string GeneratePartitionKey(LogEvent logEvent, AzureTableStorageS
3335
}
3436

3537
/// <summary>
36-
/// Automatically generates the RowKey using the timestamp
38+
/// Automatically generates the RowKey using the timestamp
3739
/// </summary>
3840
/// <param name="logEvent">the log event</param>
3941
/// <param name="options">The table storage options.</param>
@@ -48,4 +50,69 @@ public virtual string GenerateRowKey(LogEvent logEvent, AzureTableStorageSinkOpt
4850
var utcEventTime = logEvent.Timestamp.UtcDateTime;
4951
return utcEventTime.GenerateRowKey();
5052
}
53+
54+
55+
56+
/// <summary>
57+
/// Generates the PartitionKey based on the logEvent timestamp
58+
/// </summary>
59+
/// <param name="utcEventTime">The UTC event time.</param>
60+
/// <param name="roundSpan">The round span.</param>
61+
/// <returns>
62+
/// The Generated PartitionKey
63+
/// </returns>
64+
/// <remarks>
65+
/// The partition key based on the Timestamp rounded to the nearest 5 min
66+
/// </remarks>
67+
public static string GeneratePartitionKey(DateTime utcEventTime, TimeSpan? roundSpan = null)
68+
{
69+
var span = roundSpan ?? TimeSpan.FromMinutes(5);
70+
var roundedEvent = Round(utcEventTime, span);
71+
72+
// create a 19 character String for reverse chronological ordering.
73+
return $"{DateTime.MaxValue.Ticks - roundedEvent.Ticks:D19}";
74+
}
75+
76+
/// <summary>
77+
/// Generates the RowKey using the timestamp
78+
/// </summary>
79+
/// <param name="utcEventTime">The UTC event time.</param>
80+
/// <returns>
81+
/// The generated RowKey
82+
/// </returns>
83+
public static string GenerateRowKey(DateTime utcEventTime)
84+
{
85+
// create a reverse chronological ordering date
86+
var targetTicks = DateTime.MaxValue.Ticks - utcEventTime.Ticks;
87+
88+
// add incrementing value to ensure unique
89+
int padding = Next();
90+
91+
return $"{targetTicks:D19}{padding:D4}";
92+
}
93+
94+
/// <summary>
95+
/// Rounds the specified date.
96+
/// </summary>
97+
/// <param name="date">The date to round.</param>
98+
/// <param name="span">The span.</param>
99+
/// <returns>The rounded date</returns>
100+
public static DateTime Round(DateTime date, TimeSpan span)
101+
{
102+
long ticks = (date.Ticks + (span.Ticks / 2) + 1) / span.Ticks;
103+
return new DateTime(ticks * span.Ticks);
104+
}
105+
106+
107+
private static int _counter = new Random().Next(_minCounter, _maxCounter);
108+
109+
private const int _minCounter = 1;
110+
private const int _maxCounter = 9999;
111+
112+
private static int Next()
113+
{
114+
Interlocked.Increment(ref _counter);
115+
return Interlocked.CompareExchange(ref _counter, _minCounter, _maxCounter);
116+
}
117+
51118
}

src/Serilog.Sinks.AzureTableStorage/Sinks/AzureTableStorage/Extensions/DateTimeExtensions.cs

+4-9
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ namespace Serilog.Sinks.AzureTableStorage.Extensions;
77
/// </summary>
88
public static class DateTimeExtensions
99
{
10+
1011
/// <summary>
1112
/// Generates the PartitionKey based on the logEvent timestamp
1213
/// </summary>
@@ -20,11 +21,7 @@ public static class DateTimeExtensions
2021
/// </remarks>
2122
public static string GeneratePartitionKey(this DateTime utcEventTime, TimeSpan? roundSpan = null)
2223
{
23-
var span = roundSpan ?? TimeSpan.FromMinutes(5);
24-
var roundedEvent = Round(utcEventTime, span);
25-
26-
// create a 19 character String for reverse chronological ordering.
27-
return $"{DateTime.MaxValue.Ticks - roundedEvent.Ticks:D19}";
24+
return DefaultKeyGenerator.GeneratePartitionKey(utcEventTime, roundSpan);
2825
}
2926

3027
/// <summary>
@@ -36,8 +33,7 @@ public static string GeneratePartitionKey(this DateTime utcEventTime, TimeSpan?
3633
/// </returns>
3734
public static string GenerateRowKey(this DateTime utcEventTime)
3835
{
39-
// create a 19 character String for reverse chronological ordering.
40-
return $"{DateTime.MaxValue.Ticks - utcEventTime.Ticks:D19}";
36+
return DefaultKeyGenerator.GenerateRowKey(utcEventTime);
4137
}
4238

4339
/// <summary>
@@ -48,7 +44,6 @@ public static string GenerateRowKey(this DateTime utcEventTime)
4844
/// <returns>The rounded date</returns>
4945
public static DateTime Round(this DateTime date, TimeSpan span)
5046
{
51-
long ticks = (date.Ticks + (span.Ticks / 2) + 1) / span.Ticks;
52-
return new DateTime(ticks * span.Ticks);
47+
return DefaultKeyGenerator.Round(date, span);
5348
}
5449
}

0 commit comments

Comments
 (0)