Skip to content

Commit ccde8de

Browse files
andrewlockandrewlock
committed
Update GcMemoryLoadCalculator to use the TotalProcessorCount
1 parent 7544a5e commit ccde8de

2 files changed

Lines changed: 89 additions & 41 deletions

File tree

‎tracer/src/Datadog.Trace/RuntimeMetrics/GcMemoryLoadCalculator.cs‎

Lines changed: 42 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ internal static class GcMemoryLoadCalculator
3535
// high_memory_load_th is fixed for the lifetime of the GC configuration it was resolved from, so the
3636
// configured override (if any) only needs to be read once.
3737
private static readonly Lazy<int?> ConfiguredHighMemoryLoadPercent = new(ReadConfiguredHighMemoryLoadPercent);
38+
private static readonly Func<int?> GetTotalProcessorCount = () => TotalProcessorCount.Value;
3839

3940
private static bool _unableToResolveLogged;
4041

@@ -43,53 +44,59 @@ internal static class GcMemoryLoadCalculator
4344
/// </summary>
4445
public static double? TryGetMemoryLoadPercentage(in GCMemoryInfo info)
4546
{
46-
// ProcessorCount is incorrect here, we need the number of processors on the machine, not the number visible to the process
4747
return TryCalculate(
4848
info.MemoryLoadBytes,
4949
info.HighMemoryLoadThresholdBytes,
5050
info.TotalAvailableMemoryBytes,
5151
ConfiguredHighMemoryLoadPercent.Value,
52-
Environment.ProcessorCount);
52+
GetTotalProcessorCount);
5353
}
5454

5555
[TestingAndPrivateOnly]
56-
internal static double? TryCalculate(long memoryLoadBytes, long highMemoryLoadThresholdBytes, long totalAvailableMemoryBytes, int? configuredHighPercent, int processorCount)
56+
internal static double? TryCalculate(long memoryLoadBytes, long highMemoryLoadThresholdBytes, long totalAvailableMemoryBytes, int? configuredHighPercent, Func<int?> getTotalProcessorCount)
5757
{
5858
if (highMemoryLoadThresholdBytes <= 0 || totalAvailableMemoryBytes <= 0)
5959
{
6060
// HighMemoryLoadThresholdBytes is 0 before the first GC has run, so we can't calculate anything
6161
return null;
6262
}
6363

64-
var highPercent = ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent, processorCount);
64+
var highPercent = ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent, getTotalProcessorCount);
65+
if (highPercent is null)
66+
{
67+
return null;
68+
}
6569

66-
// heap_hard_limit (TotalAvailableMemoryBytes) can never exceed total_physical_mem. If the total we'd
67-
// imply from our resolved threshold is smaller than TotalAvailableMemoryBytes, the threshold is wrong -
68-
// bail out rather than publish a skewed value.
69-
var impliedTotalPhysicalMem = highMemoryLoadThresholdBytes * 100.0 / highPercent;
70+
// heap_hard_limit (TotalAvailableMemoryBytes) can never exceed total_physical_mem. If the implied total
71+
// from our resolved threshold is smaller than TotalAvailableMemoryBytes, then we got something wrong in our
72+
// calculations, so bail out rather than publish a skewed value.
73+
// This should never be violated, it's just a safety check
74+
var impliedTotalPhysicalMem = highMemoryLoadThresholdBytes * 100.0 / highPercent.Value;
7075
if (impliedTotalPhysicalMem < totalAvailableMemoryBytes * 0.99)
7176
{
7277
if (!Volatile.Read(ref _unableToResolveLogged))
7378
{
7479
Volatile.Write(ref _unableToResolveLogged, true);
75-
Log.Debug<long, long, long, int?, int>(
76-
"Unable to resolve GC memory load percentage (MemoryLoadBytes={MemoryLoadBytes}, HighMemoryLoadThresholdBytes={HighMemoryLoadThresholdBytes}, TotalAvailableMemoryBytes={TotalAvailableMemoryBytes}, ConfiguredHighPercent={ConfiguredHighPercent}, ProcessorCount={ProcessorCount})",
77-
memoryLoadBytes,
78-
highMemoryLoadThresholdBytes,
79-
totalAvailableMemoryBytes,
80-
configuredHighPercent,
81-
processorCount);
80+
Log.Warning(
81+
"Unable to resolve GC memory load percentage, implied total {ImpliedTotal} is less than total available bytes {TotalAvailableMemoryBytes} (MemoryLoadBytes={MemoryLoadBytes}, HighMemoryLoadThresholdBytes={HighMemoryLoadThresholdBytes}, ConfiguredHighPercent={ConfiguredHighPercent})",
82+
[
83+
impliedTotalPhysicalMem,
84+
totalAvailableMemoryBytes,
85+
memoryLoadBytes,
86+
highMemoryLoadThresholdBytes,
87+
configuredHighPercent
88+
]);
8289
}
8390

8491
return null;
8592
}
8693

87-
var memoryLoad = Math.Round(memoryLoadBytes * (double)highPercent / highMemoryLoadThresholdBytes);
94+
var memoryLoad = Math.Round(memoryLoadBytes * (double)highPercent.Value / highMemoryLoadThresholdBytes);
8895
return Math.Min(100d, Math.Max(0d, memoryLoad));
8996
}
9097

9198
[TestingAndPrivateOnly]
92-
internal static int ResolveHighMemoryLoadThresholdPercent(long highMemoryLoadThresholdBytes, int? configuredHighPercent, int processorCount)
99+
internal static int? ResolveHighMemoryLoadThresholdPercent(long highMemoryLoadThresholdBytes, int? configuredHighPercent, Func<int?> getTotalProcessorCount)
93100
{
94101
// We need to recreate this flow from the GC: https://github.com/dotnet/runtime/blob/2cc068d0008c898c67578f2868bd5b17a64c6366/src/coreclr/gc/init.cpp#L1488C59-L1519
95102

@@ -107,6 +114,24 @@ internal static int ResolveHighMemoryLoadThresholdPercent(long highMemoryLoadThr
107114
return 90;
108115
}
109116

117+
// If we know we're > 80GB, but we can't get the processor count, then we can't accurately
118+
// calculate the high memory load threshold percent
119+
if (getTotalProcessorCount() is not { } processorCount)
120+
{
121+
// If that processor count couldn't be reliably determined, we don't guess, we bail out.
122+
if (!Volatile.Read(ref _unableToResolveLogged))
123+
{
124+
Volatile.Write(ref _unableToResolveLogged, true);
125+
Log.Warning(
126+
"Unable to resolve GC memory load percentage: total machine processor count is unknown (HighMemoryLoadThresholdBytes={HighMemoryLoadThresholdBytes}, ConfiguredHighPercent={ConfiguredHighPercent})",
127+
highMemoryLoadThresholdBytes,
128+
configuredHighPercent);
129+
}
130+
131+
return null;
132+
}
133+
134+
// Calculating from https://github.com/dotnet/runtime/blob/2cc068d0008c898c67578f2868bd5b17a64c6366/src/coreclr/gc/init.cpp#L1508
110135
var availableMemThreshold = Math.Min(10, 3 + (int)(47f / Math.Max(1, processorCount)));
111136
return 100 - availableMemThreshold;
112137
}

‎tracer/test/Datadog.Trace.Tests/RuntimeMetrics/GcMemoryLoadCalculatorTests.cs‎

Lines changed: 47 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ public class GcMemoryLoadCalculatorTests
1919
private const long Total8GiB = 8L * 1024 * 1024 * 1024;
2020
private const long Total79GiB = 79L * 1024 * 1024 * 1024;
2121
private const long Total96GiB = 96L * 1024 * 1024 * 1024;
22+
private static readonly Func<int?> GetTotalProcessorCount = () => 4;
2223

2324
[Fact]
2425
public void Calculate_NoHardLimit_RecoversTrueLoad()
@@ -27,7 +28,12 @@ public void Calculate_NoHardLimit_RecoversTrueLoad()
2728
var totalAvailableMemoryBytes = Total8GiB;
2829
var memoryLoadBytes = Encode(Total8GiB, 42);
2930

30-
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: null, processorCount: 4);
31+
var result = GcMemoryLoadCalculator.TryCalculate(
32+
memoryLoadBytes,
33+
highMemoryLoadThresholdBytes,
34+
totalAvailableMemoryBytes,
35+
configuredHighPercent: null,
36+
GetTotalProcessorCount);
3137

3238
result.Should().Be(42);
3339
}
@@ -46,7 +52,7 @@ public void Calculate_DefaultContainerHardLimit_RecoversTrueLoad(int loadPercent
4652
var totalAvailableMemoryBytes = Encode(Total8GiB, 75);
4753
var memoryLoadBytes = Encode(Total8GiB, loadPercent);
4854

49-
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: null, processorCount: 4);
55+
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: null, GetTotalProcessorCount);
5056

5157
result.Should().Be(loadPercent);
5258
}
@@ -62,7 +68,7 @@ public void Calculate_ConfiguredViaHexEnvVar_ClampsToNinetyNine()
6268
var totalAvailableMemoryBytes = Encode(Total8GiB, 50);
6369
var memoryLoadBytes = Encode(Total8GiB, 42);
6470

65-
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent, processorCount: 4);
71+
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent, GetTotalProcessorCount);
6672

6773
result.Should().Be(42);
6874
}
@@ -80,7 +86,7 @@ public void Calculate_ConfiguredViaRuntimeConfigKnob_ParsesDecimal()
8086
var totalAvailableMemoryBytes = Encode(Total8GiB, 50);
8187
var memoryLoadBytes = Encode(Total8GiB, 42);
8288

83-
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent, processorCount: 4);
89+
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent, GetTotalProcessorCount);
8490

8591
result.Should().Be(42);
8692
}
@@ -193,42 +199,59 @@ public void ParseAppContextHighMemPercent_ResolvesExpectedValue(object? appConte
193199
result.Should().Be(expected);
194200
}
195201

196-
[Fact]
197-
public void ResolveHighMemoryLoadThresholdPercent_HostAtOrAboveEightyGiB_UsesProcessorAdjustedDefault()
202+
[Theory]
203+
[InlineData(96, 97, 48, 97)] // min(10, 3 + (int)(47 / 48)) == 3 -> th == 97
204+
[InlineData(82, 90, 4, 90)] // min(10, 3 + (int)(47 / 4)) == min(10, 14) == 10 -> th == 90
205+
[InlineData(96, 97, null, null)] // needs the host-wide processor count; if it couldn't be reliably determined, bail out
206+
public void ResolveHighMemoryLoadThresholdPercent_HostAtOrAboveEightyGiB_DependsOnProcessorCount(int totalGiB, int thresholdPercent, int? processorCount, int? expected)
198207
{
199-
var highMemoryLoadThresholdBytes = Encode(Total96GiB, 97);
208+
var totalBytes = totalGiB * 1024L * 1024 * 1024;
209+
var highMemoryLoadThresholdBytes = Encode(totalBytes, thresholdPercent);
200210

201-
// min(10, 3 + (int)(47 / 48)) == 3 -> th == 97
202-
var result = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent: null, processorCount: 48);
211+
var result = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent: null, () => processorCount);
203212

204-
result.Should().Be(97);
213+
result.Should().Be(expected);
214+
}
215+
216+
[Theory]
217+
[InlineData(4, 90)]
218+
[InlineData(null, 90)] // the flat-90 branch never needs the processor count, so an unknown count shouldn't stop it resolving
219+
public void ResolveHighMemoryLoadThresholdPercent_HostJustBelowEightyGiB_UsesFixedDefaultRegardlessOfProcessorCount(int? processorCount, int? expected)
220+
{
221+
var highMemoryLoadThresholdBytes = Encode(Total79GiB, 90);
222+
223+
var result = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent: null, () => processorCount);
224+
225+
result.Should().Be(expected);
205226
}
206227

207228
[Fact]
208-
public void ResolveHighMemoryLoadThresholdPercent_HostAtOrAboveEightyGiB_LowProcessorCount_CapsAtNinety()
229+
public void ResolveHighMemoryLoadThresholdPercent_ConfiguredOverride_UnknownProcessorCountStillResolves()
209230
{
210-
var highMemoryLoadThresholdBytes = Encode(Total82GiB, 90);
231+
// A configured override never needs the processor count either.
232+
var highMemoryLoadThresholdBytes = Encode(Total96GiB, 70);
211233

212-
// min(10, 3 + (int)(47 / 4)) == min(10, 14) == 10 -> th == 90
213-
var result = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent: null, processorCount: 4);
234+
var result = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent: 70, () => null);
214235

215-
result.Should().Be(90);
236+
result.Should().Be(70);
216237
}
217238

218239
[Fact]
219-
public void ResolveHighMemoryLoadThresholdPercent_HostJustBelowEightyGiB_UsesFixedDefault()
240+
public void Calculate_HostAtOrAboveEightyGiB_UnknownProcessorCount_ReturnsNull()
220241
{
221-
var highMemoryLoadThresholdBytes = Encode(Total79GiB, 90);
242+
var highMemoryLoadThresholdBytes = Encode(Total96GiB, 97);
243+
var totalAvailableMemoryBytes = Total96GiB;
244+
var memoryLoadBytes = Encode(Total96GiB, 42);
222245

223-
var result = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent: null, processorCount: 4);
246+
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: null, () => null);
224247

225-
result.Should().Be(90);
248+
result.Should().BeNull();
226249
}
227250

228251
[Fact]
229252
public void Calculate_HighMemoryLoadThresholdBytesIsZero_ReturnsNull()
230253
{
231-
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes: 500, highMemoryLoadThresholdBytes: 0, totalAvailableMemoryBytes: Total8GiB, configuredHighPercent: null, processorCount: 4);
254+
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes: 500, highMemoryLoadThresholdBytes: 0, totalAvailableMemoryBytes: Total8GiB, configuredHighPercent: null, GetTotalProcessorCount);
232255

233256
result.Should().BeNull();
234257
}
@@ -238,7 +261,7 @@ public void Calculate_PathologicalHardLimitExceedsImpliedPhysicalMemory_ReturnsN
238261
{
239262
// A tiny highMemoryLoadThresholdBytes next to a huge totalAvailableMemoryBytes can never be consistent -
240263
// TotalAvailableMemoryBytes (heap_hard_limit) can never exceed total_physical_mem.
241-
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes: 500, highMemoryLoadThresholdBytes: 1000, totalAvailableMemoryBytes: Total8GiB, configuredHighPercent: null, processorCount: 4);
264+
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes: 500, highMemoryLoadThresholdBytes: 1000, totalAvailableMemoryBytes: Total8GiB, configuredHighPercent: null, GetTotalProcessorCount);
242265

243266
result.Should().BeNull();
244267
}
@@ -259,7 +282,7 @@ public void Calculate_ExplicitHardLimitWhoseRatioLooksLikeACleanPercentage_Recov
259282
var totalAvailableMemoryBytes = Encode(Total8GiB, 93.75);
260283
var memoryLoadBytes = Encode(Total8GiB, loadPercent);
261284

262-
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: null, processorCount: 4);
285+
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: null, GetTotalProcessorCount);
263286

264287
result.Should().Be(loadPercent);
265288
}
@@ -273,7 +296,7 @@ public void Calculate_ConfiguredOverrideWinsUnderAHardLimit()
273296
var totalAvailableMemoryBytes = Encode(Total8GiB, 75);
274297
var memoryLoadBytes = Encode(Total8GiB, 42);
275298

276-
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: 70, processorCount: 4);
299+
var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: 70, GetTotalProcessorCount);
277300

278301
result.Should().Be(42);
279302
}
@@ -321,7 +344,7 @@ public void DriftCanary_MeasuredDefaultHighMemoryLoadThresholdMatchesProductionP
321344
var measured = TryMeasureHighMemoryLoadThresholdPercent(info.HighMemoryLoadThresholdBytes, info.TotalAvailableMemoryBytes);
322345
Skip.If(measured is null, "Could not measure high_memory_load_th on this host (a hard limit may be in play despite no knob being detected, or the C#/C++ rounding didn't round-trip).");
323346

324-
var predicted = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(info.HighMemoryLoadThresholdBytes, configuredHighPercent: null, Environment.ProcessorCount);
347+
var predicted = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(info.HighMemoryLoadThresholdBytes, configuredHighPercent: null, () => TotalProcessorCount.Value);
325348

326349
measured.Should().Be(predicted);
327350
// The ratio-inference removed from production, kept only as a measurement tool for the drift canary above:

0 commit comments

Comments
 (0)