diff --git a/tracer/src/Datadog.Trace/Configuration/PlatformKeys.DotNet.cs b/tracer/src/Datadog.Trace/Configuration/PlatformKeys.DotNet.cs index 2aae6046d57e..966a0f106b96 100644 --- a/tracer/src/Datadog.Trace/Configuration/PlatformKeys.DotNet.cs +++ b/tracer/src/Datadog.Trace/Configuration/PlatformKeys.DotNet.cs @@ -33,4 +33,18 @@ internal static partial class PlatformKeys /// Program data folder /// public const string ProgramData = "ProgramData"; + + /// + /// Sets the GC's "high memory load" threshold percent (clamped to 99 by the runtime). Parsed as + /// hexadecimal by the runtime (see GCToEEInterface::GetIntConfigValue), and takes precedence + /// over the System.GC.HighMemoryPercent runtimeconfig knob, which is parsed using C-style base + /// detection (0x/0X prefix for hexadecimal, a leading 0 for octal, otherwise decimal - + /// see Configuration::GetKnobULONGLONGValue). + /// + public const string DotNetGCHighMemPercent = "DOTNET_GCHighMemPercent"; + + /// + /// Legacy alias for , also parsed as hexadecimal. + /// + public const string ComPlusGCHighMemPercent = "COMPlus_GCHighMemPercent"; } diff --git a/tracer/src/Datadog.Trace/RuntimeMetrics/DiagnosticsMetricsRuntimeMetricsListener.cs b/tracer/src/Datadog.Trace/RuntimeMetrics/DiagnosticsMetricsRuntimeMetricsListener.cs index b77fb1ef8d3e..13fa863403d4 100644 --- a/tracer/src/Datadog.Trace/RuntimeMetrics/DiagnosticsMetricsRuntimeMetricsListener.cs +++ b/tracer/src/Datadog.Trace/RuntimeMetrics/DiagnosticsMetricsRuntimeMetricsListener.cs @@ -120,25 +120,12 @@ public void Refresh() } // memory load - // This is attempting to emulate the GcGlobalHeapHistory.MemoryLoad event details - // That value is calculated using - // - `current_gc_data_global->mem_pressure` (src/coreclr/gc/gc.cpp#L3288) - // - which fetches the value set via `history->mem_pressure = entry_memory_load` (src/coreclr/gc/gc.cpp#L7912) - // - which is set by calling `gc_heap::get_memory_info()` (src/coreclr/gc/gc.cpp#L29438) - // - which then calls GCToOSInterface::GetMemoryStatus(...) which has platform-specific implementations - // - On linux, memory_load is calculated differently depending if there's a restriction (src/coreclr/gc/unix/gcenv.unix.cpp#L1191) - // - Physical Memory Used / Limit - // - (g_totalPhysicalMemSize - GetAvailablePhysicalMemory()) / total - // - On Windows, memory_load is calculated differently depending if there's a restriction (src/coreclr/gc/unix/gcenv.windows.cpp#L1000) - // - Working Set Size / Limit - // - GlobalMemoryStatusEx -> (ullTotalVirtual - ullAvailVirtual) * 100.0 / (float)ms.ullTotalVirtual - // - // We try to roughly emulate that using the info in gcInfo: - var availableBytes = gcInfo.TotalAvailableMemoryBytes; - - if (availableBytes > 0) + // GCMemoryInfo.MemoryLoadBytes and GCMemoryInfo.HighMemoryLoadThresholdBytes are both scaled by the + // GC's total_physical_mem, but TotalAvailableMemoryBytes switches to heap_hard_limit whenever a GC + // hard limit is in play, so getting the memory load is not simple + if (GcMemoryLoadCalculator.TryGetMemoryLoadPercentage(gcInfo) is { } memoryLoad) { - statsd.Gauge(MetricsNames.GcMemoryLoad, (double)gcInfo.MemoryLoadBytes * 100.0 / availableBytes); + statsd.Gauge(MetricsNames.GcMemoryLoad, memoryLoad); } } else diff --git a/tracer/src/Datadog.Trace/RuntimeMetrics/GcMemoryLoadCalculator.cs b/tracer/src/Datadog.Trace/RuntimeMetrics/GcMemoryLoadCalculator.cs new file mode 100644 index 000000000000..71acb342b13c --- /dev/null +++ b/tracer/src/Datadog.Trace/RuntimeMetrics/GcMemoryLoadCalculator.cs @@ -0,0 +1,300 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// +#if NET6_0_OR_GREATER + +#nullable enable + +using System; +using System.Threading; +using Datadog.Trace.Configuration; +using Datadog.Trace.Logging; +using Datadog.Trace.SourceGenerators; +using Datadog.Trace.Util; + +namespace Datadog.Trace.RuntimeMetrics; + +/// +/// Recovers the true GC memory-load percentage (0-100) from . +/// and are both +/// scaled by the GC's total_physical_mem, but switches +/// to heap_hard_limit whenever a GC hard limit is in play (e.g. a memory-limited container without explicit +/// GC configuration, where the runtime defaults the limit to 75% of physical memory). +/// See GCHeap::GetMemoryInfo in src/coreclr/gc/gc.cpp. +/// +internal static class GcMemoryLoadCalculator +{ + // gc_heap::compute_memory_settings() only applies its ">= 80GB" branch above this threshold. + // The value here is pre-scaled by the default high-memory-load percentage (90%) so the comparison + // below is a plain integer comparison, not a division. + private const long EightyGiBBytesAt90Percent = 80L * 1024 * 1024 * 1024 * 9 / 10; + + private static readonly IDatadogLogger Log = DatadogLogging.GetLoggerFor(typeof(GcMemoryLoadCalculator)); + + // high_memory_load_th is fixed for the lifetime of the GC configuration it was resolved from, so the + // configured override (if any) only needs to be read once. + private static readonly Lazy ConfiguredHighMemoryLoadPercent = new(ReadConfiguredHighMemoryLoadPercent); + private static readonly Func GetTotalProcessorCount = () => TotalProcessorCount.Value; + + private static bool _unableToResolveLogged; + + /// + /// Gets the GC memory load as a 0-100 percentage, or null if it cannot be reliably determined. + /// + public static double? TryGetMemoryLoadPercentage(in GCMemoryInfo info) + { + return TryCalculate( + info.MemoryLoadBytes, + info.HighMemoryLoadThresholdBytes, + info.TotalAvailableMemoryBytes, + ConfiguredHighMemoryLoadPercent.Value, + GetTotalProcessorCount); + } + + [TestingAndPrivateOnly] + internal static double? TryCalculate(long memoryLoadBytes, long highMemoryLoadThresholdBytes, long totalAvailableMemoryBytes, int? configuredHighPercent, Func getTotalProcessorCount) + { + if (highMemoryLoadThresholdBytes <= 0 || totalAvailableMemoryBytes <= 0) + { + // HighMemoryLoadThresholdBytes is 0 before the first GC has run, so we can't calculate anything + return null; + } + + var highPercent = ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent, getTotalProcessorCount); + if (highPercent is null) + { + return null; + } + + // heap_hard_limit (TotalAvailableMemoryBytes) can never exceed total_physical_mem. If the implied total + // from our resolved threshold is smaller than TotalAvailableMemoryBytes, then we got something wrong in our + // calculations, so bail out rather than publish a skewed value. + // This should never be violated, it's just a safety check + var impliedTotalPhysicalMem = highMemoryLoadThresholdBytes * 100.0 / highPercent.Value; + if (impliedTotalPhysicalMem < totalAvailableMemoryBytes * 0.99) + { + if (!Volatile.Read(ref _unableToResolveLogged)) + { + Volatile.Write(ref _unableToResolveLogged, true); + Log.Warning( + "Unable to resolve GC memory load percentage, implied total {ImpliedTotal} is less than total available bytes {TotalAvailableMemoryBytes} (MemoryLoadBytes={MemoryLoadBytes}, HighMemoryLoadThresholdBytes={HighMemoryLoadThresholdBytes}, ConfiguredHighPercent={ConfiguredHighPercent})", + [ + impliedTotalPhysicalMem, + totalAvailableMemoryBytes, + memoryLoadBytes, + highMemoryLoadThresholdBytes, + configuredHighPercent + ]); + } + + return null; + } + + var memoryLoad = Math.Round(memoryLoadBytes * (double)highPercent.Value / highMemoryLoadThresholdBytes); + return Math.Min(100d, Math.Max(0d, memoryLoad)); + } + + [TestingAndPrivateOnly] + internal static int? ResolveHighMemoryLoadThresholdPercent(long highMemoryLoadThresholdBytes, int? configuredHighPercent, Func getTotalProcessorCount) + { + // We need to recreate this flow from the GC: https://github.com/dotnet/runtime/blob/2cc068d0008c898c67578f2868bd5b17a64c6366/src/coreclr/gc/init.cpp#L1488C59-L1519 + + // An explicit override (from env/AppContext) always wins + if (configuredHighPercent is { } configured) + { + return configured; + } + + // Otherwise, the threshold should be the runtime's default formula. + // Since the resolved percentage is always >= 90, the _implied_ total + // here is always >= total_physical_mem, so "implied < 80GiB" implies "total_physical_mem < 80GiB". + if (highMemoryLoadThresholdBytes < EightyGiBBytesAt90Percent) + { + return 90; + } + + // If we know we're > 80GB, but we can't get the processor count, then we can't accurately + // calculate the high memory load threshold percent + if (getTotalProcessorCount() is not { } processorCount) + { + // If that processor count couldn't be reliably determined, we don't guess, we bail out. + if (!Volatile.Read(ref _unableToResolveLogged)) + { + Volatile.Write(ref _unableToResolveLogged, true); + Log.Warning( + "Unable to resolve GC memory load percentage: total machine processor count is unknown (HighMemoryLoadThresholdBytes={HighMemoryLoadThresholdBytes}, ConfiguredHighPercent={ConfiguredHighPercent})", + highMemoryLoadThresholdBytes, + configuredHighPercent); + } + + return null; + } + + // Calculating from https://github.com/dotnet/runtime/blob/2cc068d0008c898c67578f2868bd5b17a64c6366/src/coreclr/gc/init.cpp#L1508 + var availableMemThreshold = Math.Min(10, 3 + (int)(47f / Math.Max(1, processorCount))); + return 100 - availableMemThreshold; + } + + [TestingAndPrivateOnly] + internal static int? ParseEnvHighMemPercent(ReadOnlySpan envValue) + { + // GCToEEInterface::GetIntConfigValue reads the env knob via u16_strtoui64(value, &end, 16) - a 64-bit + // parse on every platform (Windows: _wcstoui64, Unix: PAL__wcstoui64 -> strtoull) - and treats ERANGE + // (overflow past ulong.MaxValue) as "not specified at all" rather than clamping + if (!TryParseCStyleUnsignedInteger(envValue, numberBase: 16, out var parsed, out var overflowed) || overflowed) + { + return null; + } + + return ToGcHighMemPercent(parsed); + } + + [TestingAndPrivateOnly] + internal static int? ParseAppContextHighMemPercent(object? appContextValue) + { + // runtimeconfig properties always reach AppContext as strings - anything else was set by user code + // after startup, so the GC never saw it + if (appContextValue is not string stringValue) + { + return null; + } + + // Configuration::GetKnobULONGLONGValue reads the runtimeconfig knob via u16_strtoui64(value, nullptr, 0) + // (base 0 - decimal, or hex/octal by prefix) and ignores the return value's ERANGE (errno) entirely, so + // an out-of-range value saturates to UINT64_MAX and gets clamped to 99 below + TryParseCStyleUnsignedInteger(stringValue.AsSpan(), numberBase: 0, out var parsed, out _); + return ToGcHighMemPercent(parsed); + } + + // compute_memory_settings() (see the pinned source reference on EightyGiBBytesAt90Percent above) reads the + // config value into a 32-bit integer, so the high bits are silently dropped, then treats 0 as "not + // configured" and clamps the result to 99. + private static int? ToGcHighMemPercent(ulong configValue) + { + var truncated = (uint)configValue; + return truncated == 0 ? null : (int)Math.Min(99u, truncated); + } + + // Emulates the C runtime's strtoull(value, &end, numberBase), which both u16_strtoui64(..., 16) (env, + // via GetIntConfigValue) and u16_strtoui64(..., 0) (runtimeconfig, via GetKnobULONGLONGValue) build on. + // Parsing stops at the first invalid character rather than requiring the whole span to match, ERANGE + // (overflow past ulong.MaxValue) is reported separately rather than failing the parse, and a leading '-' + // negates the result within the unsigned range rather than being rejected - but only when the magnitude + // didn't itself overflow: strtoull() saturates to ULLONG_MAX on ERANGE *before* the sign would be applied, + // so an overflowing negative value must stay saturated, not get negated down to 1. + private static bool TryParseCStyleUnsignedInteger(ReadOnlySpan value, int numberBase, out ulong result, out bool overflowed) + { + result = 0; + overflowed = false; + + var i = 0; + while (i < value.Length && char.IsWhiteSpace(value[i])) + { + i++; + } + + var negative = false; + if (i < value.Length && (value[i] == '+' || value[i] == '-')) + { + negative = value[i] == '-'; + i++; + } + + if (numberBase is 16 or 0 && + i + 1 < value.Length && value[i] == '0' && (value[i + 1] is 'x' or 'X') && + i + 2 < value.Length && HexDigitValue(value[i + 2]) >= 0) + { + numberBase = 16; + i += 2; + } + else if (numberBase == 0) + { + numberBase = i < value.Length && value[i] == '0' ? 8 : 10; + } + + var digitsConsumed = 0; + for (; i < value.Length; i++) + { + var digit = HexDigitValue(value[i]); + if (digit < 0 || digit >= numberBase) + { + break; + } + + digitsConsumed++; + + if (overflowed) + { + continue; + } + + if (result > (ulong.MaxValue - (ulong)digit) / (ulong)numberBase) + { + overflowed = true; + result = ulong.MaxValue; + continue; + } + + result = (result * (ulong)numberBase) + (ulong)digit; + } + + if (digitsConsumed == 0) + { + result = 0; + return false; + } + + if (negative && !overflowed) + { + result = unchecked(0UL - result); + } + + return true; + + // Returns the digit's value for 0-9/a-f/A-F, or -1 if the char isn't a hex digit. + static int HexDigitValue(char c) => c switch + { + >= '0' and <= '9' => c - '0', + >= 'a' and <= 'f' => c - 'a' + 10, + >= 'A' and <= 'F' => c - 'A' + 10, + _ => -1, + }; + } + + [TestingAndPrivateOnly] + internal static int? ReadConfiguredHighMemoryLoadPercent() + { + // Read the configs defined here: https://github.com/dotnet/runtime/blob/2cc068d0008c898c67578f2868bd5b17a64c6366/src/coreclr/gc/gcconfig.h#L100 + try + { + var envValue = EnvironmentHelpers.GetEnvironmentVariable(PlatformKeys.DotNetGCHighMemPercent) + ?? EnvironmentHelpers.GetEnvironmentVariable(PlatformKeys.ComPlusGCHighMemPercent); + + // The runtime checks the environment variable first (gcenv.ee.cpp: GetGCHighMemPercent()). If it's + // present at all - even "0", which means "unset" - it wins outright and the runtimeconfig knob below is + // never consulted, so an explicit-but-unset env var can't fall back to a configured runtimeconfig value. + if (envValue is not null) + { + return ParseEnvHighMemPercent(envValue.AsSpan()); + } + } + catch (Exception ex) + { + Log.Error(ex, "Error reading configured GC high memory load percent"); + } + + try + { + // The runtimeconfig knob (System.GC.HighMemoryPercent) is only consulted if the environment variable is unset. + return ParseAppContextHighMemPercent(AppContext.GetData("System.GC.HighMemoryPercent")); + } + catch (Exception ex) + { + Log.Debug(ex, "Error reading System.GC.HighMemoryPercent from AppContext"); + } + + return null; + } +} +#endif diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs index 39d1958f353b..9bfe07885c36 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs @@ -55,6 +55,41 @@ public async Task DiagnosticsMetricsApiSubmitsMetrics() EnvironmentHelper.EnableDefaultTransport(); await RunTest(); } + + [SkippableFact] + [Trait("Category", "EndToEnd")] + [Trait("RunOnWindows", "True")] + [Trait("SupportsInstrumentationVerification", "True")] + public async Task DiagnosticsMetricsApiMemoryLoad_StaysWithinValidRange_UnderGcHardLimit() + { + // DOTNET_GCHeapHardLimitPercent (parsed as hex, so "19" == 0x19 == 25%) sets + // gc_heap::heap_hard_limit != 0 (similar to a memory-limited container). + SetEnvironmentVariable(ConfigurationKeys.RuntimeMetricsDiagnosticsMetricsApiEnabled, "1"); + SetEnvironmentVariable("DOTNET_GCHeapHardLimitPercent", "19"); + EnvironmentHelper.EnableDefaultTransport(); + + using var agent = EnvironmentHelper.GetMockAgent(useStatsD: true); + using var processResult = await RunSampleAndWaitForExit(agent); + var requests = agent.StatsdRequests; + requests.Should().NotBeEmpty(); + + var metrics = requests.SelectMany(x => x.Split('\n')).ToList(); + var memoryLoadSamples = metrics + .Where(m => m.StartsWith(MetricsNames.GcMemoryLoad + ":", StringComparison.Ordinal)) + .Select( + m => + { + var separator = m.IndexOf(':'); + var endIndex = m.IndexOf('|', separator + 1); + return double.Parse(m.Substring(separator + 1, endIndex - separator - 1)); + }) + .ToList(); + + memoryLoadSamples.Should().NotBeEmpty(); + memoryLoadSamples.Should().OnlyContain(v => v > 0 && v <= 100); + + agent.Exceptions.Should().BeEmpty(); + } #endif [SkippableFact] diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/DiagnosticMetricsRuntimeMetricsListenerTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/DiagnosticMetricsRuntimeMetricsListenerTests.cs index 3148b11bdb36..cad0329e9f20 100644 --- a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/DiagnosticMetricsRuntimeMetricsListenerTests.cs +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/DiagnosticMetricsRuntimeMetricsListenerTests.cs @@ -20,6 +20,7 @@ using Datadog.Trace.RuntimeMetrics; using Datadog.Trace.TestHelpers.Stats; using Datadog.Trace.Vendors.StatsdClient; +using FluentAssertions; using Moq; using Xunit; using Range = Moq.Range; @@ -43,7 +44,10 @@ public void PushEvents() // some metrics are only recorded the _second_ time this is called, to avoid skewing the results at the start, so we just check for a couple statsd.Verify(s => s.Gauge(MetricsNames.Gen0HeapSize, It.IsAny(), 1, null), Times.Once); - statsd.Verify(s => s.Gauge(MetricsNames.GcMemoryLoad, It.IsInRange(0d, 100, Range.Inclusive), It.IsAny(), null), Times.AtLeastOnce); + + var expectedMemoryLoad = GcMemoryLoadCalculator.TryGetMemoryLoadPercentage(GC.GetGCMemoryInfo()); + expectedMemoryLoad.Should().NotBeNull(); + statsd.Verify(s => s.Gauge(MetricsNames.GcMemoryLoad, expectedMemoryLoad!.Value, It.IsAny(), null), Times.AtLeastOnce); } [Fact] diff --git a/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/GcMemoryLoadCalculatorTests.cs b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/GcMemoryLoadCalculatorTests.cs new file mode 100644 index 000000000000..2672ee3c8450 --- /dev/null +++ b/tracer/test/Datadog.Trace.Tests/RuntimeMetrics/GcMemoryLoadCalculatorTests.cs @@ -0,0 +1,381 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#if NET6_0_OR_GREATER + +#nullable enable + +using System; +using Datadog.Trace.RuntimeMetrics; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.Tests.RuntimeMetrics; + +public class GcMemoryLoadCalculatorTests +{ + private const long Total8GiB = 8L * 1024 * 1024 * 1024; + private const long Total79GiB = 79L * 1024 * 1024 * 1024; + private const long Total96GiB = 96L * 1024 * 1024 * 1024; + private static readonly Func GetTotalProcessorCount = () => 4; + + [Fact] + public void Calculate_NoHardLimit_RecoversTrueLoad() + { + var highMemoryLoadThresholdBytes = Encode(Total8GiB, 90); + var totalAvailableMemoryBytes = Total8GiB; + var memoryLoadBytes = Encode(Total8GiB, 42); + + var result = GcMemoryLoadCalculator.TryCalculate( + memoryLoadBytes, + highMemoryLoadThresholdBytes, + totalAvailableMemoryBytes, + configuredHighPercent: null, + GetTotalProcessorCount); + + result.Should().Be(42); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(10)] + [InlineData(42)] + [InlineData(80)] + [InlineData(99)] + [InlineData(100)] + public void Calculate_DefaultContainerHardLimit_RecoversTrueLoad(int loadPercent) + { + var highMemoryLoadThresholdBytes = Encode(Total8GiB, 90); + var totalAvailableMemoryBytes = Encode(Total8GiB, 75); + var memoryLoadBytes = Encode(Total8GiB, loadPercent); + + var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: null, GetTotalProcessorCount); + + result.Should().Be(loadPercent); + } + + [Fact] + public void Calculate_ConfiguredViaHexEnvVar_ClampsToNinetyNine() + { + // "90" parsed as hex is 144, which the runtime clamps to 99 + var configuredHighPercent = GcMemoryLoadCalculator.ParseEnvHighMemPercent("90".AsSpan()); + configuredHighPercent.Should().Be(99); + + var highMemoryLoadThresholdBytes = Encode(Total8GiB, 99); + var totalAvailableMemoryBytes = Encode(Total8GiB, 50); + var memoryLoadBytes = Encode(Total8GiB, 42); + + var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent, GetTotalProcessorCount); + + result.Should().Be(42); + } + + [Fact] + public void Calculate_ConfiguredViaRuntimeConfigKnob_ParsesDecimal() + { + // The runtimeconfig knob (System.GC.HighMemoryPercent) is only consulted if the environment variable + // is unset (see ParseEnvHighMemPercent_ResolvesExpectedValue / ReadConfiguredHighMemoryLoadPercent for + // that precedence), and is parsed as decimal. + var configuredHighPercent = GcMemoryLoadCalculator.ParseAppContextHighMemPercent((object?)"95"); + configuredHighPercent.Should().Be(95); + + var highMemoryLoadThresholdBytes = Encode(Total8GiB, 95); + var totalAvailableMemoryBytes = Encode(Total8GiB, 50); + var memoryLoadBytes = Encode(Total8GiB, 42); + + var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent, GetTotalProcessorCount); + + result.Should().Be(42); + } + + [Theory] + [InlineData("0", null)] + [InlineData("1", 1)] + [InlineData("5", 5)] + [InlineData("9", 9)] + [InlineData("a", 10)] + [InlineData("A", 10)] + [InlineData("007", 7)] + [InlineData("50", 80)] + [InlineData("63", 99)] + [InlineData("64", 99)] + [InlineData("99", 99)] + [InlineData("ff", 99)] + [InlineData("FF", 99)] + [InlineData("0x50", 80)] + [InlineData("0X50", 80)] + [InlineData("0x1", 1)] + [InlineData("0x63", 99)] + [InlineData("0xFF", 99)] + [InlineData("0x0", null)] + [InlineData(" 50", 80)] + [InlineData("50 ", 80)] + [InlineData(" 63 ", 99)] + [InlineData("50xyz", 80)] + [InlineData("50.5", 80)] + [InlineData("5g", 5)] + [InlineData("63!!", 99)] + [InlineData("1,2,3", 1)] + [InlineData("", null)] + [InlineData(" ", null)] + [InlineData("xyz", null)] + [InlineData("gg", null)] + [InlineData("-", null)] + [InlineData("+", null)] + [InlineData("0x", null)] + [InlineData("+50", 80)] + [InlineData("-1", 99)] // the real runtime accepts the sign, wraps to a huge unsigned value, and clamps to 99 + [InlineData("-5", 99)] + [InlineData("-50", 99)] + [InlineData("-63", 99)] + [InlineData("-64", 99)] + [InlineData("-0", null)] + [InlineData("100000000", null)] // 0x100000000 is exactly 2^32, so the (uint32_t) truncation in init.cpp zeroes it out entirely; a huge configured value silently becomes "unconfigured." + [InlineData("100000005", 5)] // truncation isn't just "big values become 99"; it's a literal low-32-bit truncation + [InlineData("100000063", 99)] + [InlineData("-100000000", null)] + [InlineData("10000000000000000", null)] // 17+ hex digits overflow 64-bit unsigned, strtoull sets ERANGE, and GetIntConfigValue treats that as a hard failure (unconfigured), not a clamp. + [InlineData("1ffffffffffffffff", null)] + [InlineData("-1ffffffffffffffff", null)] // ERANGE is a hard failure regardless of sign on the env path - unlike the runtimeconfig path below, it never reaches the sign/clamp logic at all + [InlineData("ffffffffffffffff", 99)] // exactly UINT64_MAX, fits in 64 bits with no ERANGE, so it takes the normal wraparound-then-clamp path + public void ParseEnvHighMemPercent_ResolvesExpectedValue(string? envValue, int? expected) + { + var result = GcMemoryLoadCalculator.ParseEnvHighMemPercent(envValue.AsSpan()); + result.Should().Be(expected); + } + + [Theory] + [InlineData("1", 1)] + [InlineData("50", 50)] + [InlineData("63", 63)] + [InlineData("90", 90)] + [InlineData("98", 98)] + [InlineData("99", 99)] + [InlineData("100", 99)] + [InlineData("153", 99)] + [InlineData("0", null)] + [InlineData("0x50", 80)] + [InlineData("0X1E", 30)] + [InlineData("0x63", 99)] + [InlineData("0xFF", 99)] + [InlineData("0x0", null)] + [InlineData("010", 8)] + [InlineData("045", 37)] + [InlineData("077", 63)] + [InlineData("090", null)] + [InlineData("099", null)] + [InlineData("008", null)] + [InlineData("08", null)] + [InlineData("-1", 99)] + [InlineData("-50", 99)] + [InlineData("+50", 50)] + [InlineData("-0", null)] + [InlineData("-0x50", 99)] + [InlineData(" 50", 50)] + [InlineData("50 ", 50)] + [InlineData("50abc", 50)] + [InlineData("50.5", 50)] + [InlineData("63,64", 63)] + [InlineData("true", null)] + [InlineData("", null)] + [InlineData(" ", null)] + [InlineData("abc", null)] + [InlineData("-", null)] + [InlineData("+", null)] + [InlineData("0x", null)] + [InlineData("ffffffffffffffff", null)] + [InlineData("99999999999999999999", 99)] + [InlineData("18446744073709551616", 99)] + [InlineData("4294967296", null)] + [InlineData("4294967301", 5)] + [InlineData("4294967395", 99)] + [InlineData("-4294967296", null)] + [InlineData("-010", 99)] + [InlineData("-18446744073709551616", 99)] // magnitude is exactly 2^64: overflows before the sign is applied, so strtoull saturates to ULLONG_MAX (not negated down to 1) and that clamps to 99 + [InlineData("-99999999999999999999", 99)] // same, with a magnitude far past 2^64 + [InlineData("-0xffffffffffffffffff", 99)] // same, via the hex prefix instead of decimal + public void ParseAppContextHighMemPercent_ResolvesExpectedValue(object? appContextValue, int? expected) + { + var result = GcMemoryLoadCalculator.ParseAppContextHighMemPercent(appContextValue); + result.Should().Be(expected); + } + + [Theory] + [InlineData(96, 97, 48, 97)] // min(10, 3 + (int)(47 / 48)) == 3 -> th == 97 + [InlineData(82, 90, 4, 90)] // min(10, 3 + (int)(47 / 4)) == min(10, 14) == 10 -> th == 90 + [InlineData(96, 97, null, null)] // needs the host-wide processor count; if it couldn't be reliably determined, bail out + public void ResolveHighMemoryLoadThresholdPercent_HostAtOrAboveEightyGiB_DependsOnProcessorCount(int totalGiB, int thresholdPercent, int? processorCount, int? expected) + { + var totalBytes = totalGiB * 1024L * 1024 * 1024; + var highMemoryLoadThresholdBytes = Encode(totalBytes, thresholdPercent); + + var result = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent: null, () => processorCount); + + result.Should().Be(expected); + } + + [Theory] + [InlineData(4, 90)] + [InlineData(null, 90)] // the flat-90 branch never needs the processor count, so an unknown count shouldn't stop it resolving + public void ResolveHighMemoryLoadThresholdPercent_HostJustBelowEightyGiB_UsesFixedDefaultRegardlessOfProcessorCount(int? processorCount, int? expected) + { + var highMemoryLoadThresholdBytes = Encode(Total79GiB, 90); + + var result = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent: null, () => processorCount); + + result.Should().Be(expected); + } + + [Fact] + public void ResolveHighMemoryLoadThresholdPercent_ConfiguredOverride_UnknownProcessorCountStillResolves() + { + // A configured override never needs the processor count either. + var highMemoryLoadThresholdBytes = Encode(Total96GiB, 70); + + var result = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(highMemoryLoadThresholdBytes, configuredHighPercent: 70, () => null); + + result.Should().Be(70); + } + + [Fact] + public void Calculate_HostAtOrAboveEightyGiB_UnknownProcessorCount_ReturnsNull() + { + var highMemoryLoadThresholdBytes = Encode(Total96GiB, 97); + var totalAvailableMemoryBytes = Total96GiB; + var memoryLoadBytes = Encode(Total96GiB, 42); + + var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: null, () => null); + + result.Should().BeNull(); + } + + [Fact] + public void Calculate_HighMemoryLoadThresholdBytesIsZero_ReturnsNull() + { + var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes: 500, highMemoryLoadThresholdBytes: 0, totalAvailableMemoryBytes: Total8GiB, configuredHighPercent: null, GetTotalProcessorCount); + + result.Should().BeNull(); + } + + [Fact] + public void Calculate_PathologicalHardLimitExceedsImpliedPhysicalMemory_ReturnsNull() + { + // A tiny highMemoryLoadThresholdBytes next to a huge totalAvailableMemoryBytes can never be consistent - + // TotalAvailableMemoryBytes (heap_hard_limit) can never exceed total_physical_mem. + var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes: 500, highMemoryLoadThresholdBytes: 1000, totalAvailableMemoryBytes: Total8GiB, configuredHighPercent: null, GetTotalProcessorCount); + + result.Should().BeNull(); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(42)] + [InlineData(60)] + [InlineData(99)] + [InlineData(100)] + public void Calculate_ExplicitHardLimitWhoseRatioLooksLikeACleanPercentage_RecoversTrueLoad(int loadPercent) + { + // A hard limit of 93.75% (15/16) of physical memory, over the default 90% threshold, inverts to a + // byte-exact 96, so if we try to do anything funny by messing with the numbers, and looking for + // rational values, it would fall through here + var highMemoryLoadThresholdBytes = Encode(Total8GiB, 90); + var totalAvailableMemoryBytes = Encode(Total8GiB, 93.75); + var memoryLoadBytes = Encode(Total8GiB, loadPercent); + + var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: null, GetTotalProcessorCount); + + result.Should().Be(loadPercent); + } + + [Fact] + public void Calculate_ConfiguredOverrideWinsUnderAHardLimit() + { + // Configured override (70) under a 75% hard limit: neither the observed ratio (90) nor the runtime's + // default formula (90) gives the right answer - only the configured override does. + var highMemoryLoadThresholdBytes = Encode(Total8GiB, 70); + var totalAvailableMemoryBytes = Encode(Total8GiB, 75); + var memoryLoadBytes = Encode(Total8GiB, 42); + + var result = GcMemoryLoadCalculator.TryCalculate(memoryLoadBytes, highMemoryLoadThresholdBytes, totalAvailableMemoryBytes, configuredHighPercent: 70, GetTotalProcessorCount); + + result.Should().Be(42); + } + + // Drift canary: exploits the fact that on a host with no GC hard limit, heap_hard_limit and total_physical_mem coincide, so + // TryMeasureHighMemoryLoadThresholdPercent can *measure* high_memory_load_th directly from live GCMemoryInfo - + // no assumptions involved. Comparing that measurement against production's ResolveHighMemoryLoadThresholdPercent + // means that if a future runtime ever changes the default formula, the measurement follows it while + // production's hardcoded copy does not, and the test fails. On a sub-80 GiB agent this only pins the flat 90 branch; + // the processor-adjusted branch is covered only by the synthetic unit tests above, which can't drift-detect. + [SkippableFact] + public void DriftCanary_MeasuredDefaultHighMemoryLoadThresholdMatchesProductionPrediction() + { + var info = GC.GetGCMemoryInfo(); + Skip.If(info.HighMemoryLoadThresholdBytes <= 0 || info.TotalAvailableMemoryBytes <= 0, "No GC has run yet in this process"); + + // Any GC hard-limit knob whose presence means heap_hard_limit no longer equals total_physical_mem, which is + // what TryMeasureHighMemoryLoadThresholdPercent's inversion assumes. gc.cpp sets heap_hard_limit to the sum + // of the per-generation limits, so the SOH/LOH/POH variants have to be covered too, not just the combined + // knob. The public (AppContext/runtimeconfig) key for each of these is "System.GC." + the env suffix (see + // gcconfig.h) - GCHighMemPercent doesn't create a hard limit at all (it only shifts the threshold), so it's + // excluded here and instead fed through production's own parsers below. + var gcHardLimitKnobsThatInvalidateTheMeasurement = new[] + { + ("GCHeapHardLimit", "System.GC.HeapHardLimit"), + ("GCHeapHardLimitPercent", "System.GC.HeapHardLimitPercent"), + ("GCHeapHardLimitSOH", "System.GC.HeapHardLimitSOH"), + ("GCHeapHardLimitLOH", "System.GC.HeapHardLimitLOH"), + ("GCHeapHardLimitPOH", "System.GC.HeapHardLimitPOH"), + ("GCHeapHardLimitSOHPercent", "System.GC.HeapHardLimitSOHPercent"), + ("GCHeapHardLimitLOHPercent", "System.GC.HeapHardLimitLOHPercent"), + ("GCHeapHardLimitPOHPercent", "System.GC.HeapHardLimitPOHPercent"), + }; + + foreach (var (envSuffix, appContextKey) in gcHardLimitKnobsThatInvalidateTheMeasurement) + { + // Presence only, no parsing: a mis-parse here would silently re-enable the very ratio-inference + // this canary exists to keep out of production. + var envPresent = Environment.GetEnvironmentVariable("DOTNET_" + envSuffix) is not null + || Environment.GetEnvironmentVariable("COMPlus_" + envSuffix) is not null; + envPresent.Should().BeFalse($"DOTNET_{envSuffix}/COMPlus_{envSuffix} is configured for this process; the canary only targets a host with no GC hard limit."); + AppContext.GetData(appContextKey).Should().BeNull($"AppContext key {appContextKey} is configured for this process; the canary only targets a host with no GC hard limit."); + } + + var measured = TryMeasureHighMemoryLoadThresholdPercent(info.HighMemoryLoadThresholdBytes, info.TotalAvailableMemoryBytes); + Skip.If(measured is null, "Could not measure high_memory_load_th on this host (a hard limit may be in play despite no knob being detected, or the C#/C++ rounding didn't round-trip)."); + + // Unlike the hard-limit knobs above, GCHighMemPercent doesn't invalidate the measurement - it only changes + // the expected threshold - so instead of requiring its absence, resolve it through the same production + // parsers used at runtime. + var configuredHighPercent = GcMemoryLoadCalculator.ReadConfiguredHighMemoryLoadPercent(); + var predicted = GcMemoryLoadCalculator.ResolveHighMemoryLoadThresholdPercent(info.HighMemoryLoadThresholdBytes, configuredHighPercent, () => TotalProcessorCount.Value); + + measured.Should().Be(predicted); + // The ratio-inference removed from production, kept only as a measurement tool for the drift canary above: + // on a host with no GC hard limit, totalPhysicalMemoryBytes IS total_physical_mem, so inverting + // thresholdBytes / totalPhysicalMemoryBytes and round-tripping it byte-exact really does recover + // high_memory_load_th, with no assumptions about configuration or the default formula involved. + static int? TryMeasureHighMemoryLoadThresholdPercent(long thresholdBytes, long totalPhysicalMemoryBytes) + { + var implied = (int)Math.Round((thresholdBytes / (double)totalPhysicalMemoryBytes) * 100); + if (implied is < 1 or > 99) + { + return null; + } + + var roundTrip = (long)((implied / 100.0) * totalPhysicalMemoryBytes); + return roundTrip == thresholdBytes ? implied : null; + } + } + + // Mirrors how the GC encodes a percentage into bytes (compute_memory_settings, src/coreclr/gc/init.cpp as of + // writing - see the pinned source reference on GcMemoryLoadCalculator.EightyGiBBytesAt90Percent): + // `(uint64_t)(pct / 100 * total_physical_mem)`. Each test builds inputs the same way the runtime would, then + // asserts we decode the original percentage back. + private static long Encode(long total, double percent) => (long)((percent / 100.0) * total); +} +#endif