Read container tags hash from agent

tracer/missing-nullability-files.csv

@@ -123,7 +123,6 @@ src/Datadog.Trace/HttpOverStreams/IHttpContent.cs
 src/Datadog.Trace/Iast/ITaintedMap.cs
 src/Datadog.Trace/Iast/SourceType.cs
 src/Datadog.Trace/PlatformHelpers/AspNetCoreHttpRequestHandler.cs
-src/Datadog.Trace/PlatformHelpers/ContainerMetadata.cs
 src/Datadog.Trace/PlatformHelpers/ServiceFabric.cs
 src/Datadog.Trace/Processors/ITagProcessor.cs
 src/Datadog.Trace/Processors/NormalizerTraceProcessor.cs

tracer/src/Datadog.Trace.Tools.Runner/Utils.cs

@@ -21,6 +21,7 @@
 using Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
 using Datadog.Trace.Configuration.Telemetry;
 using Datadog.Trace.Logging;
+using Datadog.Trace.PlatformHelpers;
 using Datadog.Trace.Tools.Runner.Gac;
 using Datadog.Trace.Util;
 using Spectre.Console;
@@ -420,6 +421,7 @@ public static async Task<AgentConfiguration> CheckAgentConnectionAsync(string ag
             Log.Debug("Creating DiscoveryService for: {AgentUri}", settings.Manager.InitialExporterSettings.AgentUri);
             var discoveryService = DiscoveryService.CreateUnmanaged(
                 settings.Manager.InitialExporterSettings,
+                ContainerMetadata.Instance,
                 tcpTimeout: TimeSpan.FromSeconds(5),
                 initialRetryDelayMs: 200,
                 maxRetryDelayMs: 1000,

tracer/src/Datadog.Trace/Agent/DiscoveryService/DiscoveryService.cs

@@ -14,6 +14,7 @@
 using Datadog.Trace.Configuration;
 using Datadog.Trace.HttpOverStreams;
 using Datadog.Trace.Logging;
+using Datadog.Trace.PlatformHelpers;
 using Datadog.Trace.SourceGenerators;
 using Datadog.Trace.Util;
 using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;
@@ -46,6 +47,7 @@ internal sealed class DiscoveryService : IDiscoveryService
         private readonly object _lock = new();
         private readonly Task _discoveryTask;
         private readonly IDisposable? _settingSubscription;
+        private readonly ContainerMetadata _containerMetadata;
         private IApiRequestFactory _apiRequestFactory;
         private AgentConfiguration? _configuration;
         private string? _configurationHash;
@@ -54,18 +56,19 @@ internal sealed class DiscoveryService : IDiscoveryService
 
         public DiscoveryService(
             TracerSettings.SettingsManager settings,
+            ContainerMetadata containerMetadata,
             TimeSpan tcpTimeout,
             int initialRetryDelayMs,
             int maxRetryDelayMs,
             int recheckIntervalMs)
-            : this(CreateApiRequestFactory(settings.InitialExporterSettings, tcpTimeout), initialRetryDelayMs, maxRetryDelayMs, recheckIntervalMs)
+            : this(CreateApiRequestFactory(settings.InitialExporterSettings, containerMetadata.ContainerId, tcpTimeout), containerMetadata, initialRetryDelayMs, maxRetryDelayMs, recheckIntervalMs)
         {
             // Create as a "managed" service that can update the request factory
             _settingSubscription = settings.SubscribeToChanges(changes =>
             {
                 if (changes.UpdatedExporter is { } exporter)
                 {
-                    var newFactory = CreateApiRequestFactory(exporter, tcpTimeout);
+                    var newFactory = CreateApiRequestFactory(exporter, containerMetadata.ContainerId, tcpTimeout);
                     Interlocked.Exchange(ref _apiRequestFactory!, newFactory);
                 }
             });
@@ -77,11 +80,13 @@ public DiscoveryService(
         /// </summary>
         public DiscoveryService(
             IApiRequestFactory apiRequestFactory,
+            ContainerMetadata containerMetadata,
             int initialRetryDelayMs,
             int maxRetryDelayMs,
             int recheckIntervalMs)
         {
             _apiRequestFactory = apiRequestFactory;
+            _containerMetadata = containerMetadata;
             _initialRetryDelayMs = initialRetryDelayMs;
             _maxRetryDelayMs = maxRetryDelayMs;
             _recheckIntervalMs = recheckIntervalMs;
@@ -114,9 +119,10 @@ public DiscoveryService(
         /// <summary>
         /// Create a <see cref="DiscoveryService"/> instance that responds to runtime changes in settings
         /// </summary>
-        public static DiscoveryService CreateManaged(TracerSettings settings)
+        public static DiscoveryService CreateManaged(TracerSettings settings, ContainerMetadata containerMetadata)
             => new(
                 settings.Manager,
+                containerMetadata,
                 tcpTimeout: TimeSpan.FromSeconds(15),
                 initialRetryDelayMs: 500,
                 maxRetryDelayMs: 5_000,
@@ -125,9 +131,10 @@ public static DiscoveryService CreateManaged(TracerSettings settings)
         /// <summary>
         /// Create a <see cref="DiscoveryService"/> instance that does _not_ respond to runtime changes in settings
         /// </summary>
-        public static DiscoveryService CreateUnmanaged(ExporterSettings exporterSettings)
+        public static DiscoveryService CreateUnmanaged(ExporterSettings exporterSettings, ContainerMetadata containerMetadata)
             => CreateUnmanaged(
                 exporterSettings,
+                containerMetadata,
                 tcpTimeout: TimeSpan.FromSeconds(15),
                 initialRetryDelayMs: 500,
                 maxRetryDelayMs: 5_000,
@@ -138,12 +145,14 @@ public static DiscoveryService CreateUnmanaged(ExporterSettings exporterSettings
         /// </summary>
         public static DiscoveryService CreateUnmanaged(
             ExporterSettings exporterSettings,
+            ContainerMetadata containerMetadata,
             TimeSpan tcpTimeout,
             int initialRetryDelayMs,
             int maxRetryDelayMs,
             int recheckIntervalMs)
             => new(
-                CreateApiRequestFactory(exporterSettings, tcpTimeout),
+                CreateApiRequestFactory(exporterSettings, containerMetadata.ContainerId, tcpTimeout),
+                containerMetadata,
                 initialRetryDelayMs,
                 maxRetryDelayMs,
                 recheckIntervalMs);
@@ -297,6 +306,13 @@ internal bool RequireRefresh(string? currentHash, DateTimeOffset utcNow)
 
         private async Task ProcessDiscoveryResponse(IApiResponse response)
         {
+            // Extract and store container tags hash from response headers
+            var containerTagsHash = response.GetHeader(AgentHttpHeaderNames.ContainerTagsHash);
+            if (containerTagsHash != null)
+            {
+                _containerMetadata.ContainerTagsHash = containerTagsHash;
+            }
+
             // Grab the original stream
             var stream = await response.GetStreamAsync().ConfigureAwait(false);
 
@@ -440,13 +456,34 @@ public Task DisposeAsync()
             return _discoveryTask;
         }
 
-        private static IApiRequestFactory CreateApiRequestFactory(ExporterSettings exporterSettings, TimeSpan tcpTimeout)
-            => AgentTransportStrategy.Get(
+        /// <summary>
+        /// Builds the headers array for the discovery service, including the container ID if available.
+        /// Internal for testing purposes.
+        /// </summary>
+        internal static KeyValuePair<string, string>[] BuildHeaders(string? containerId)
+        {
+            if (containerId != null)
+            {
+                // if container ID is available, add it to headers
+                return
+                [
+                    ..AgentHttpHeaderNames.MinimalHeaders,
+                    new(AgentHttpHeaderNames.ContainerId, containerId),
+                ];
+            }
+
+            return AgentHttpHeaderNames.MinimalHeaders;
+        }
+
+        private static IApiRequestFactory CreateApiRequestFactory(ExporterSettings exporterSettings, string? containerId, TimeSpan tcpTimeout)
+        {
+            return AgentTransportStrategy.Get(
                 exporterSettings,
                 productName: "discovery",
                 tcpTimeout: tcpTimeout,
-                AgentHttpHeaderNames.MinimalHeaders,
-                () => new MinimalAgentHeaderHelper(),
+                BuildHeaders(containerId),
+                () => new MinimalAgentHeaderHelper(containerId),
                 uri => uri);
+        }
     }
 }

tracer/src/Datadog.Trace/AgentHttpHeaderNames.cs

@@ -43,6 +43,11 @@ internal static class AgentHttpHeaderNames
         /// </summary>
         public const string ContainerId = "Datadog-Container-ID";
 
+        /// <summary>
+        /// The hash of the container tags received from the agent.
+        /// </summary>
+        public const string ContainerTagsHash = "Datadog-Container-Tags-Hash";
+
         /// <summary>
         /// The unique identifier of the container where the traced application is running, either as the container id
         /// or the cgroup node controller's inode.

tracer/src/Datadog.Trace/Ci/TestOptimization.cs

@@ -14,6 +14,7 @@
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Logging;
 using Datadog.Trace.Pdb;
+using Datadog.Trace.PlatformHelpers;
 using Datadog.Trace.Telemetry;
 using Datadog.Trace.Util;
 using TaskExtensions = Datadog.Trace.ExtensionMethods.TaskExtensions;
@@ -248,6 +249,7 @@ public void Initialize()
             settings: Settings,
             getDiscoveryServiceFunc: static s => DiscoveryService.CreateUnmanaged(
                 s.TracerSettings.Manager.InitialExporterSettings,
+                ContainerMetadata.Instance,
                 tcpTimeout: TimeSpan.FromSeconds(5),
                 initialRetryDelayMs: 100,
                 maxRetryDelayMs: 1000,

tracer/src/Datadog.Trace/HttpOverStreams/MinimalAgentHeaderHelper.cs

@@ -11,6 +11,13 @@ namespace Datadog.Trace.HttpOverStreams;
 internal class MinimalAgentHeaderHelper : HttpHeaderHelperBase
 {
     private static string? _metadataHeaders = null;
+    private static string? _metadataHeadersWithContainerId = null;
+    private readonly string? _containerId;
+
+    public MinimalAgentHeaderHelper(string? containerId = null)
+    {
+        _containerId = containerId;
+    }
 
     protected override string MetadataHeaders
     {
@@ -22,6 +29,18 @@ protected override string MetadataHeaders
                 _metadataHeaders = string.Concat(headers);
             }
 
+            if (_containerId != null)
+            {
+                if (_metadataHeadersWithContainerId == null)
+                {
+                    // Assuming we'd always get the same container ID. The first time, we use its value to build the header,
+                    // then it's just a marker that we want the header with the containerID in it, and we don't look at its value.
+                    _metadataHeadersWithContainerId = _metadataHeaders + $"{AgentHttpHeaderNames.ContainerId}: {_containerId}{DatadogHttpValues.CrLf}";
+                }
+
+                return _metadataHeadersWithContainerId;
+            }
+
             return _metadataHeaders;
         }
     }

tracer/src/Datadog.Trace/PlatformHelpers/ContainerMetadata.NetFramework.cs

@@ -5,6 +5,8 @@
 
 #nullable enable
 
+using System.Threading;
+
 #if NETFRAMEWORK
 
 namespace Datadog.Trace.PlatformHelpers;
@@ -26,6 +28,16 @@ public ContainerMetadata(string containerId, string entityId)
         // nothing to do, just to match the other version
     }
 
+    /// <summary>
+    /// Gets or sets the container tags hash received from the agent, used by DBM/DSM
+    /// This is set when we receive a value for it in an http response from the agent
+    /// </summary>
+    public string? ContainerTagsHash
+    {
+        get => Volatile.Read(ref field);
+        set => Volatile.Write(ref field, value);
+    }
+
     /// <summary>
     /// Gets the id of the container executing the code.
     /// Return <c>null</c> if code is not executing inside a supported container.

tracer/src/Datadog.Trace/PlatformHelpers/ContainerMetadata.cs

@@ -3,6 +3,7 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
 // </copyright>
 
+#nullable enable
 #if !NETFRAMEWORK
 
 using System;
@@ -43,28 +44,39 @@ internal sealed class ContainerMetadata
 
         public static readonly ContainerMetadata Instance = new();
 
-        private readonly Lazy<string> _containerId;
-        private readonly Lazy<string> _entityId;
+        private readonly Lazy<string?> _containerId;
+        private readonly Lazy<string?> _entityId;
 
         private ContainerMetadata()
         {
-            _containerId = new Lazy<string>(GetContainerIdInternal, LazyThreadSafetyMode.ExecutionAndPublication);
-            _entityId = new Lazy<string>(() => GetEntityIdInternal(_containerId), LazyThreadSafetyMode.ExecutionAndPublication);
+            _containerId = new Lazy<string?>(GetContainerIdInternal, LazyThreadSafetyMode.ExecutionAndPublication);
+            _entityId = new Lazy<string?>(() => GetEntityIdInternal(_containerId), LazyThreadSafetyMode.ExecutionAndPublication);
         }
 
         // For use in tests only
-        public ContainerMetadata(string containerId, string entityId)
+        [TestingOnly]
+        public ContainerMetadata(string? containerId, string? entityId)
         {
-            _containerId = new Lazy<string>(() => containerId);
-            _entityId = new Lazy<string>(() => entityId);
+            _containerId = new Lazy<string?>(() => containerId);
+            _entityId = new Lazy<string?>(() => entityId);
+        }
+
+        /// <summary>
+        /// Gets or sets the container tags hash received from the agent, used by DBM/DSM
+        /// This is set when we receive a value for it in an http response from the agent
+        /// </summary>
+        public string? ContainerTagsHash
+        {
+            get => Volatile.Read(ref field);
+            set => Volatile.Write(ref field, value);
         }
 
         /// <summary>
         /// Gets the id of the container executing the code.
         /// Return <c>null</c> if code is not executing inside a supported container.
         /// </summary>
         /// <value>The container id or <c>null</c>.</value>
-        public string ContainerId
+        public string? ContainerId
         {
             get => _containerId.Value;
         }
@@ -80,7 +92,7 @@ public string ContainerId
         /// </list>
         /// </summary>
         /// <value>The entity id or <c>null</c>.</value>
-        public string EntityId
+        public string? EntityId
         {
             get => _entityId.Value;
         }
@@ -90,7 +102,7 @@ public string EntityId
         /// </summary>
         /// <param name="lines">Lines of text from a cgroup file.</param>
         /// <returns>The container id if found; otherwise, <c>null</c>.</returns>
-        public static string ParseContainerIdFromCgroupLines(IEnumerable<string> lines)
+        public static string? ParseContainerIdFromCgroupLines(IEnumerable<string> lines)
         {
             return lines.Select(ParseContainerIdFromCgroupLine)
                         .FirstOrDefault(id => !string.IsNullOrWhiteSpace(id));
@@ -101,7 +113,7 @@ public static string ParseContainerIdFromCgroupLines(IEnumerable<string> lines)
         /// </summary>
         /// <param name="line">A single line from a cgroup file.</param>
         /// <returns>The container id if found; otherwise, <c>null</c>.</returns>
-        public static string ParseContainerIdFromCgroupLine(string line)
+        public static string? ParseContainerIdFromCgroupLine(string line)
         {
             var lineMatch = Regex.Match(line, ContainerIdRegex);
 
@@ -119,7 +131,7 @@ public static string ParseContainerIdFromCgroupLine(string line)
         /// <param name="controlGroupsMountPath">Path to the cgroup mount point.</param>
         /// <param name="lines">Lines of text from a cgroup file.</param>
         /// <returns>The cgroup node controller's inode if found; otherwise, <c>null</c>.</returns>
-        public static string ExtractInodeFromCgroupLines(string controlGroupsMountPath, IEnumerable<string> lines)
+        public static string? ExtractInodeFromCgroupLines(string controlGroupsMountPath, IEnumerable<string> lines)
         {
             foreach (var line in lines)
             {
@@ -147,7 +159,7 @@ public static string ExtractInodeFromCgroupLines(string controlGroupsMountPath,
         /// </summary>
         /// <param name="line">A single line from a cgroup file.</param>
         /// <returns>The controller/cgroup-node-path pair if found; otherwise, <c>null</c>.</returns>
-        public static Tuple<string, string> ParseControllerAndPathFromCgroupLine(string line)
+        public static Tuple<string, string>? ParseControllerAndPathFromCgroupLine(string line)
         {
             var lineMatch = Regex.Match(line, CgroupRegex);
 
@@ -181,7 +193,7 @@ internal static bool TryGetInodeUsingPInvoke(string path, out long result)
                 return false;
             }
 
-            static void LogError(Exception ex, string message)
+            static void LogError(Exception? ex, string message)
             {
 #pragma warning disable DDLOG004 // Must use constant strings - disabled as it's an integer only, and only called twice in the app lifetime
                 if (EnvironmentHelpersNoLogging.IsClrProfilerAttachedSafe())
@@ -215,7 +227,7 @@ internal static bool TryGetInodeUsingStat(string path, out long result)
             }
         }
 
-        private static string GetContainerIdInternal()
+        private static string? GetContainerIdInternal()
         {
             try
             {
@@ -236,7 +248,7 @@ private static string GetContainerIdInternal()
             return null;
         }
 
-        private static string GetEntityIdInternal(Lazy<string> lazyContainerId)
+        private static string? GetEntityIdInternal(Lazy<string?> lazyContainerId)
         {
             if (lazyContainerId.Value is string containerId)
             {
@@ -252,7 +264,7 @@ private static string GetEntityIdInternal(Lazy<string> lazyContainerId)
             }
         }
 
-        private static string GetCgroupInode()
+        private static string? GetCgroupInode()
         {
             try
             {