Skip to content

Read container tags hash from agent #7893

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vandonr merged 17 commits into from
Feb 3, 2026
Merged

Read container tags hash from agent #7893

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
9a80cbf
read container tags hash from agent
vandonr Dec 2, 2025
661e5db
[IAST] Use recursive_mutex instead of CS and mutex (#7890)
daniel-romano-DD Dec 3, 2025
d565d21
make ContainerMetadata an instance class
vandonr Dec 3, 2025
1ad4b02
use constant in tests
vandonr Dec 3, 2025
37a3ac1
adapt code to instance container metadata
vandonr Dec 4, 2025
c93c3a9
Merge remote-tracking branch 'origin/master' into vandonr/process3
vandonr Jan 5, 2026
66630ec
use local instance
vandonr Jan 5, 2026
dcb297a
nit
vandonr Jan 5, 2026
653a3a5
fix integration tests
vandonr Jan 5, 2026
d9abd75
use volatile read/write
vandonr Jan 19, 2026
a5f3f8d
use collection expression
vandonr Jan 19, 2026
c18fc65
add container ID header to MinimalAgentHeaderHelper
vandonr Jan 19, 2026
4ca4fdf
Merge remote-tracking branch 'origin/master' into vandonr/process3
vandonr Jan 19, 2026
cbe2f90
nullability in container metadata
vandonr Jan 21, 2026
c0fd225
mark as testing only
vandonr Feb 3, 2026
553b1ff
remove useless line in test ctor
vandonr Feb 3, 2026
7527eb7
create new object when mutated
vandonr Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions tracer/src/Datadog.Trace.Tools.Runner/Utils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
using Datadog.Trace.Configuration.Telemetry;
using Datadog.Trace.Logging;
using Datadog.Trace.PlatformHelpers;
using Datadog.Trace.Tools.Runner.Gac;
using Datadog.Trace.Util;
using Spectre.Console;
Expand Down Expand Up @@ -420,6 +421,7 @@ public static async Task<AgentConfiguration> CheckAgentConnectionAsync(string ag
Log.Debug("Creating DiscoveryService for: {AgentUri}", settings.Manager.InitialExporterSettings.AgentUri);
var discoveryService = DiscoveryService.CreateUnmanaged(
settings.Manager.InitialExporterSettings,
ContainerMetadata.Instance,
tcpTimeout: TimeSpan.FromSeconds(5),
initialRetryDelayMs: 200,
maxRetryDelayMs: 1000,
Expand Down
52 changes: 44 additions & 8 deletions tracer/src/Datadog.Trace/Agent/DiscoveryService/DiscoveryService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using Datadog.Trace.Configuration;
using Datadog.Trace.HttpOverStreams;
using Datadog.Trace.Logging;
using Datadog.Trace.PlatformHelpers;
using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;

namespace Datadog.Trace.Agent.DiscoveryService
Expand Down Expand Up @@ -43,23 +44,25 @@ internal sealed class DiscoveryService : IDiscoveryService
private readonly object _lock = new();
private readonly Task _discoveryTask;
private readonly IDisposable? _settingSubscription;
private readonly ContainerMetadata _containerMetadata;
private IApiRequestFactory _apiRequestFactory;
private AgentConfiguration? _configuration;

public DiscoveryService(
TracerSettings.SettingsManager settings,
ContainerMetadata containerMetadata,
TimeSpan tcpTimeout,
int initialRetryDelayMs,
int maxRetryDelayMs,
int recheckIntervalMs)
: this(CreateApiRequestFactory(settings.InitialExporterSettings, tcpTimeout), initialRetryDelayMs, maxRetryDelayMs, recheckIntervalMs)
: this(CreateApiRequestFactory(settings.InitialExporterSettings, containerMetadata.ContainerId, tcpTimeout), containerMetadata, initialRetryDelayMs, maxRetryDelayMs, recheckIntervalMs)
{
// Create as a "managed" service that can update the request factory
_settingSubscription = settings.SubscribeToChanges(changes =>
{
if (changes.UpdatedExporter is { } exporter)
{
var newFactory = CreateApiRequestFactory(exporter, tcpTimeout);
var newFactory = CreateApiRequestFactory(exporter, containerMetadata.ContainerId, tcpTimeout);
Interlocked.Exchange(ref _apiRequestFactory!, newFactory);
}
});
Expand All @@ -71,11 +74,13 @@ public DiscoveryService(
/// </summary>
public DiscoveryService(
IApiRequestFactory apiRequestFactory,
ContainerMetadata containerMetadata,
int initialRetryDelayMs,
int maxRetryDelayMs,
int recheckIntervalMs)
{
_apiRequestFactory = apiRequestFactory;
_containerMetadata = containerMetadata;
_initialRetryDelayMs = initialRetryDelayMs;
_maxRetryDelayMs = maxRetryDelayMs;
_recheckIntervalMs = recheckIntervalMs;
Expand Down Expand Up @@ -105,9 +110,10 @@ public DiscoveryService(
/// <summary>
/// Create a <see cref="DiscoveryService"/> instance that responds to runtime changes in settings
/// </summary>
public static DiscoveryService CreateManaged(TracerSettings settings)
public static DiscoveryService CreateManaged(TracerSettings settings, ContainerMetadata containerMetadata)
=> new(
settings.Manager,
containerMetadata,
tcpTimeout: TimeSpan.FromSeconds(15),
initialRetryDelayMs: 500,
maxRetryDelayMs: 5_000,
Expand All @@ -116,9 +122,10 @@ public static DiscoveryService CreateManaged(TracerSettings settings)
/// <summary>
/// Create a <see cref="DiscoveryService"/> instance that does _not_ respond to runtime changes in settings
/// </summary>
public static DiscoveryService CreateUnmanaged(ExporterSettings exporterSettings)
public static DiscoveryService CreateUnmanaged(ExporterSettings exporterSettings, ContainerMetadata containerMetadata)
=> CreateUnmanaged(
exporterSettings,
containerMetadata,
tcpTimeout: TimeSpan.FromSeconds(15),
initialRetryDelayMs: 500,
maxRetryDelayMs: 5_000,
Expand All @@ -129,12 +136,14 @@ public static DiscoveryService CreateUnmanaged(ExporterSettings exporterSettings
/// </summary>
public static DiscoveryService CreateUnmanaged(
ExporterSettings exporterSettings,
ContainerMetadata containerMetadata,
TimeSpan tcpTimeout,
int initialRetryDelayMs,
int maxRetryDelayMs,
int recheckIntervalMs)
=> new(
CreateApiRequestFactory(exporterSettings, tcpTimeout),
CreateApiRequestFactory(exporterSettings, containerMetadata.ContainerId, tcpTimeout),
containerMetadata,
initialRetryDelayMs,
maxRetryDelayMs,
recheckIntervalMs);
Expand Down Expand Up @@ -254,6 +263,13 @@ int GetNextSleepDuration(int? previousDuration) =>

private async Task ProcessDiscoveryResponse(IApiResponse response)
{
// Extract and store container tags hash from response headers
var containerTagsHash = response.GetHeader(AgentHttpHeaderNames.ContainerTagsHash);
Copy link
Copy Markdown
Member

@andrewlock andrewlock Feb 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was wondering about this code, as we're going to do the work to extract and set this hash with every response, but I think that's fine, given that we shouldn't call the discovery service so much any more since #7979 (and also given the amount of work we do later in this method)

Copy link
Copy Markdown
Contributor Author

@vandonr vandonr Feb 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, since it's not supposed to change, we could check if we already have it, and only query it then. But if you say it's fine :)

if (containerTagsHash != null)
{
_containerMetadata.ContainerTagsHash = containerTagsHash;
}

var jObject = await response.ReadAsTypeAsync<JObject>().ConfigureAwait(false);
if (jObject is null)
{
Expand Down Expand Up @@ -373,13 +389,33 @@ public Task DisposeAsync()
return _discoveryTask;
}

private static IApiRequestFactory CreateApiRequestFactory(ExporterSettings exporterSettings, TimeSpan tcpTimeout)
=> AgentTransportStrategy.Get(
/// <summary>
/// Builds the headers array for the discovery service, including the container ID if available.
/// Internal for testing purposes.
/// </summary>
internal static KeyValuePair<string, string>[] BuildHeaders(string? containerId)
{
if (containerId != null)
Copy link
Copy Markdown
Member

@andrewlock andrewlock Jan 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We never "discover" this later, right? i.e. if containerId is non null the first time this is called, it will be non-null for ever more (and never change) right?

Copy link
Copy Markdown
Contributor Author

@vandonr vandonr Jan 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes correct. Are you asking because if yes, then you'd have this code written differently, or just out of curiosity ?

Copy link
Copy Markdown
Member

@andrewlock andrewlock Feb 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, if the answer had been "yes" we would have to write this differently, because we're building this collection once and never re-evaluating. Which is fine if this always gives the same result, but not otherwise 🙂

{
// if container ID is available, add it to headers
var headers = new KeyValuePair<string, string>[AgentHttpHeaderNames.MinimalHeaders.Length + 1];
Array.Copy(AgentHttpHeaderNames.MinimalHeaders, headers, AgentHttpHeaderNames.MinimalHeaders.Length);
headers[AgentHttpHeaderNames.MinimalHeaders.Length] = new KeyValuePair<string, string>(AgentHttpHeaderNames.ContainerId, containerId);
return headers;
}

return AgentHttpHeaderNames.MinimalHeaders;
}

private static IApiRequestFactory CreateApiRequestFactory(ExporterSettings exporterSettings, string? containerId, TimeSpan tcpTimeout)
{
return AgentTransportStrategy.Get(
exporterSettings,
productName: "discovery",
tcpTimeout: tcpTimeout,
AgentHttpHeaderNames.MinimalHeaders,
BuildHeaders(containerId),
() => new MinimalAgentHeaderHelper(),
Copy link
Copy Markdown
Member

@andrewlock andrewlock Jan 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you change one of these, you need to change both of them. Different ones are use for different transports. Which means you likely need a custom MinimalAgentHeaderHelper() here, something like this:

containerId is null
  ? () => new MinimalAgentHeaderHelper()
  : () => new MinimalWithContainerIdHelper()

I know, it sucks 😅

Copy link
Copy Markdown
Contributor Author

@vandonr vandonr Jan 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm, yeah, not great, I have to do a different type because we use the instance to access a static field :/
btw, do you know why we don't use a normal Lazy in that helper ?

Copy link
Copy Markdown
Contributor Author

@vandonr vandonr Jan 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I really didn't like the idea of duplicating the whole class, so i went another route, and made MinimalAgentHeaderHelper return a different value depending on whether it was given a container ID in the ctor.

uri => uri);
}
}
}
5 changes: 5 additions & 0 deletions tracer/src/Datadog.Trace/AgentHttpHeaderNames.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ internal static class AgentHttpHeaderNames
/// </summary>
public const string ContainerId = "Datadog-Container-ID";

/// <summary>
/// The hash of the container tags received from the agent.
/// </summary>
public const string ContainerTagsHash = "Datadog-Container-Tags-Hash";

/// <summary>
/// The unique identifier of the container where the traced application is running, either as the container id
/// or the cgroup node controller's inode.
Expand Down
2 changes: 2 additions & 0 deletions tracer/src/Datadog.Trace/Ci/TestOptimization.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using Datadog.Trace.Configuration;
using Datadog.Trace.Logging;
using Datadog.Trace.Pdb;
using Datadog.Trace.PlatformHelpers;
using Datadog.Trace.Telemetry;
using Datadog.Trace.Util;
using TaskExtensions = Datadog.Trace.ExtensionMethods.TaskExtensions;
Expand Down Expand Up @@ -248,6 +249,7 @@ public void Initialize()
settings: Settings,
getDiscoveryServiceFunc: static s => DiscoveryService.CreateUnmanaged(
s.TracerSettings.Manager.InitialExporterSettings,
ContainerMetadata.Instance,
tcpTimeout: TimeSpan.FromSeconds(5),
initialRetryDelayMs: 100,
maxRetryDelayMs: 1000,
Expand Down
6 changes: 6 additions & 0 deletions tracer/src/Datadog.Trace/PlatformHelpers/ContainerMetadata.NetFramework.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ public ContainerMetadata(string containerId, string entityId)
// nothing to do, just to match the other version
}

/// <summary>
/// Gets or sets the container tags hash received from the agent, used by DBM/DSM
/// This is set when we receive a value for it in an http response from the agent
/// </summary>
public string? ContainerTagsHash { get; set; }

/// <summary>
/// Gets the id of the container executing the code.
/// Return <c>null</c> if code is not executing inside a supported container.
Expand Down
6 changes: 6 additions & 0 deletions tracer/src/Datadog.Trace/PlatformHelpers/ContainerMetadata.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ public ContainerMetadata(string containerId, string entityId)
_entityId = new Lazy<string>(() => entityId);
}

/// <summary>
/// Gets or sets the container tags hash received from the agent, used by DBM/DSM
/// This is set when we receive a value for it in an http response from the agent
/// </summary>
public string ContainerTagsHash { get; set; }

/// <summary>
/// Gets the id of the container executing the code.
/// Return <c>null</c> if code is not executing inside a supported container.
Expand Down
5 changes: 3 additions & 2 deletions tracer/src/Datadog.Trace/TracerManagerFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using Datadog.Trace.Logging;
using Datadog.Trace.Logging.DirectSubmission;
using Datadog.Trace.Logging.TracerFlare;
using Datadog.Trace.PlatformHelpers;
using Datadog.Trace.RemoteConfigurationManagement;
using Datadog.Trace.RemoteConfigurationManagement.Transport;
using Datadog.Trace.RuntimeMetrics;
Expand Down Expand Up @@ -285,8 +286,8 @@ protected virtual IAgentWriter GetAgentWriter(TracerSettings settings, IStatsdMa
}

internal virtual IDiscoveryService GetDiscoveryService(TracerSettings settings)
=> settings.AgentFeaturePollingEnabled ?
DiscoveryService.CreateManaged(settings) :
=> settings.AgentFeaturePollingEnabled ? DiscoveryService.CreateManaged(settings, ContainerMetadata.Instance)
:
NullDiscoveryService.Instance;
}
}
3 changes: 2 additions & 1 deletion tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
using Datadog.Trace.DogStatsd;
using Datadog.Trace.LibDatadog;
using Datadog.Trace.LibDatadog.DataPipeline;
using Datadog.Trace.PlatformHelpers;
using Datadog.Trace.Telemetry;
using Datadog.Trace.TestHelpers;
using Datadog.Trace.TestHelpers.Stats;
Expand Down Expand Up @@ -76,7 +77,7 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport)

var sampleRateResponses = new ConcurrentQueue<Dictionary<string, float>>();

var discovery = DiscoveryService.CreateUnmanaged(tracerSettings.Manager.InitialExporterSettings);
var discovery = DiscoveryService.CreateUnmanaged(tracerSettings.Manager.InitialExporterSettings, new ContainerMetadata(containerId: null, entityId: null));
var statsd = new NoOpStatsd();

// We have to replace the agent writer so that we can intercept the sample rate responses
Expand Down
6 changes: 3 additions & 3 deletions tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public async Task SendsStatsWithProcessing_Normalizer()
{ ConfigurationKeys.TraceDataPipelineEnabled, "false" },
});

var discovery = DiscoveryService.CreateUnmanaged(settings.Manager.InitialExporterSettings);
var discovery = DiscoveryService.CreateUnmanaged(settings.Manager.InitialExporterSettings, new ContainerMetadata(null, null));
// Note: we are explicitly _not_ using a using here, as we dispose it ourselves manually at a specific point
// and this was easiest to retrofit without changing the test structure too much.
var tracer = TracerHelper.Create(settings, agentWriter: null, sampler: null, scopeManager: null, statsd: null, discoveryService: discovery);
Expand Down Expand Up @@ -205,7 +205,7 @@ public async Task SendsStatsWithProcessing_Obfuscator()
{ ConfigurationKeys.TraceDataPipelineEnabled, "false" },
});

var discovery = DiscoveryService.CreateUnmanaged(settings.Manager.InitialExporterSettings);
var discovery = DiscoveryService.CreateUnmanaged(settings.Manager.InitialExporterSettings, new ContainerMetadata(null, null));
// Note: we are explicitly _not_ using a using here, as we dispose it ourselves manually at a specific point
// and this was easiest to retrofit without changing the test structure too much.
var tracer = TracerHelper.Create(settings, agentWriter: null, sampler: null, scopeManager: null, statsd: null, discoveryService: discovery);
Expand Down Expand Up @@ -366,7 +366,7 @@ private async Task SendStatsHelper(bool statsComputationEnabled, bool expectStat
{ ConfigurationKeys.TraceDataPipelineEnabled, "false" },
}));

var discovery = DiscoveryService.CreateUnmanaged(settings.Manager.InitialExporterSettings);
var discovery = DiscoveryService.CreateUnmanaged(settings.Manager.InitialExporterSettings, new ContainerMetadata(null, null));
// Note: we are explicitly _not_ using a using here, as we dispose it ourselves manually at a specific point
// and this was easiest to retrofit without changing the test structure too much.
var tracer = TracerHelper.Create(settings, agentWriter: null, sampler: null, scopeManager: null, statsd: null, discoveryService: discovery);
Expand Down
9 changes: 6 additions & 3 deletions tracer/test/Datadog.Trace.TestHelpers/TransportHelpers/TestApiRequest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,19 @@ internal class TestApiRequest : IApiRequest
private readonly int _statusCode;
private readonly string _responseContent;
private readonly string _responseContentType;
private readonly Dictionary<string, string> _responseHeaders;

public TestApiRequest(
Uri endpoint,
int statusCode = 200,
string responseContent = "{}",
string responseContentType = "application/json")
string responseContentType = "application/json",
Dictionary<string, string> responseHeaders = null)
{
_statusCode = statusCode;
_responseContent = responseContent;
_responseContentType = responseContentType;
_responseHeaders = responseHeaders;
Endpoint = endpoint;
}

Expand All @@ -46,7 +49,7 @@ public void AddHeader(string name, string value)

public virtual Task<IApiResponse> GetAsync()
{
var response = new TestApiResponse(_statusCode, _responseContent, _responseContentType);
var response = new TestApiResponse(_statusCode, _responseContent, _responseContentType, _responseHeaders);
Responses.Add(response);

return Task.FromResult((IApiResponse)response);
Expand All @@ -57,7 +60,7 @@ public Task<IApiResponse> PostAsync(ArraySegment<byte> traces, string contentTyp

public virtual Task<IApiResponse> PostAsync(ArraySegment<byte> bytes, string contentType, string contentEncoding)
{
var response = new TestApiResponse(_statusCode, _responseContent, _responseContentType);
var response = new TestApiResponse(_statusCode, _responseContent, _responseContentType, _responseHeaders);
Responses.Add(response);
ContentType = contentType;

Expand Down
7 changes: 5 additions & 2 deletions tracer/test/Datadog.Trace.TestHelpers/TransportHelpers/TestApiResponse.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
// </copyright>

using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
Expand All @@ -14,12 +15,14 @@ namespace Datadog.Trace.TestHelpers.TransportHelpers;
internal class TestApiResponse : IApiResponse
{
private readonly string _body;
private readonly Dictionary<string, string> _headers;

public TestApiResponse(int statusCode, string body, string contentType)
public TestApiResponse(int statusCode, string body, string contentType, Dictionary<string, string> headers = null)
{
StatusCode = statusCode;
_body = body;
ContentTypeHeader = contentType;
_headers = headers ?? new Dictionary<string, string>();
}

public string ContentTypeHeader { get; }
Expand All @@ -38,7 +41,7 @@ public void Dispose()
{
}

public string GetHeader(string headerName) => throw new NotImplementedException();
public string GetHeader(string headerName) => _headers.TryGetValue(headerName, out var value) ? value : null;

public Task<Stream> GetStreamAsync()
{
Expand Down
Loading
Loading