Fix runtime metrics

Author: andrewlock

tracer/src/Datadog.Trace/RuntimeMetrics/DiagnosticsMetricsRuntimeMetricsListener.cs

@@ -120,25 +120,12 @@ public void Refresh()
             }
 
             // memory load
-            // This is attempting to emulate the GcGlobalHeapHistory.MemoryLoad event details
-            // That value is calculated using
-            // - `current_gc_data_global->mem_pressure` (src/coreclr/gc/gc.cpp#L3288)
-            // - which fetches the value set via `history->mem_pressure = entry_memory_load` (src/coreclr/gc/gc.cpp#L7912)
-            // - which is set by calling `gc_heap::get_memory_info()` (src/coreclr/gc/gc.cpp#L29438)
-            // - which then calls GCToOSInterface::GetMemoryStatus(...) which has platform-specific implementations
-            // - On linux, memory_load is calculated differently depending if there's a restriction (src/coreclr/gc/unix/gcenv.unix.cpp#L1191)
-            //   - Physical Memory Used / Limit
-            //   - (g_totalPhysicalMemSize - GetAvailablePhysicalMemory()) / total
-            // - On Windows, memory_load is calculated differently depending if there's a restriction (src/coreclr/gc/unix/gcenv.windows.cpp#L1000)
-            //   - Working Set Size / Limit
-            //   - GlobalMemoryStatusEx -> (ullTotalVirtual - ullAvailVirtual) * 100.0 / (float)ms.ullTotalVirtual
-            //
-            // We try to roughly emulate that using the info in gcInfo:
-            var availableBytes = gcInfo.TotalAvailableMemoryBytes;
-
-            if (availableBytes > 0)
+            // GCMemoryInfo.MemoryLoadBytes and GCMemoryInfo.HighMemoryLoadThresholdBytes are both scaled by the
+            // GC's total_physical_mem, but TotalAvailableMemoryBytes switches to heap_hard_limit whenever a GC
+            // hard limit is in play, so getting the memory load is not simple
+            if (GcMemoryLoadCalculator.TryGetMemoryLoadPercentage(gcInfo) is { } memoryLoad)
             {
-                statsd.Gauge(MetricsNames.GcMemoryLoad, (double)gcInfo.MemoryLoadBytes * 100.0 / availableBytes);
+                statsd.Gauge(MetricsNames.GcMemoryLoad, memoryLoad);
             }
         }
         else

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs

@@ -55,6 +55,41 @@ public async Task DiagnosticsMetricsApiSubmitsMetrics()
             EnvironmentHelper.EnableDefaultTransport();
             await RunTest();
         }
+
+        [SkippableFact]
+        [Trait("Category", "EndToEnd")]
+        [Trait("RunOnWindows", "True")]
+        [Trait("SupportsInstrumentationVerification", "True")]
+        public async Task DiagnosticsMetricsApiMemoryLoad_StaysWithinValidRange_UnderGcHardLimit()
+        {
+            // DOTNET_GCHeapHardLimitPercent (parsed as hex, so "19" == 0x19 == 25%) sets
+            // gc_heap::heap_hard_limit != 0 (similar to a memory-limited container).
+            SetEnvironmentVariable(ConfigurationKeys.RuntimeMetricsDiagnosticsMetricsApiEnabled, "1");
+            SetEnvironmentVariable("DOTNET_GCHeapHardLimitPercent", "19");
+            EnvironmentHelper.EnableDefaultTransport();
+
+            using var agent = EnvironmentHelper.GetMockAgent(useStatsD: true);
+            using var processResult = await RunSampleAndWaitForExit(agent);
+            var requests = agent.StatsdRequests;
+            requests.Should().NotBeEmpty();
+
+            var metrics = requests.SelectMany(x => x.Split('\n')).ToList();
+            var memoryLoadSamples = metrics
+                                    .Where(m => m.StartsWith(MetricsNames.GcMemoryLoad + ":", StringComparison.Ordinal))
+                                    .Select(
+                                         m =>
+                                         {
+                                             var separator = m.IndexOf(':');
+                                             var endIndex = m.IndexOf('|', separator + 1);
+                                             return double.Parse(m.Substring(separator + 1, endIndex - separator - 1));
+                                         })
+                                    .ToList();
+
+            memoryLoadSamples.Should().NotBeEmpty();
+            memoryLoadSamples.Should().OnlyContain(v => v > 0 && v <= 100);
+
+            agent.Exceptions.Should().BeEmpty();
+        }
 #endif
 
         [SkippableFact]

tracer/test/Datadog.Trace.Tests/RuntimeMetrics/DiagnosticMetricsRuntimeMetricsListenerTests.cs

@@ -20,6 +20,7 @@
 using Datadog.Trace.RuntimeMetrics;
 using Datadog.Trace.TestHelpers.Stats;
 using Datadog.Trace.Vendors.StatsdClient;
+using FluentAssertions;
 using Moq;
 using Xunit;
 using Range = Moq.Range;
@@ -43,7 +44,10 @@ public void PushEvents()
 
         // some metrics are only recorded the _second_ time this is called, to avoid skewing the results at the start, so we just check for a couple
         statsd.Verify(s => s.Gauge(MetricsNames.Gen0HeapSize, It.IsAny<double>(), 1, null), Times.Once);
-        statsd.Verify(s => s.Gauge(MetricsNames.GcMemoryLoad, It.IsInRange(0d, 100, Range.Inclusive), It.IsAny<double>(), null), Times.AtLeastOnce);
+
+        var expectedMemoryLoad = GcMemoryLoadCalculator.TryGetMemoryLoadPercentage(GC.GetGCMemoryInfo());
+        expectedMemoryLoad.Should().NotBeNull();
+        statsd.Verify(s => s.Gauge(MetricsNames.GcMemoryLoad, expectedMemoryLoad!.Value, It.IsAny<double>(), null), Times.AtLeastOnce);
     }
 
     [Fact]