Reproduce and fix reparent issue with Ocelot

Author: bouwkast

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Http/HttpClient/SocketsHttpHandler/OcelotMessageInvokerPoolIntegration.cs

@@ -73,12 +73,15 @@ internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget
             return new CallTargetReturn<TReturn>(returnValue);
         }
 
-        // On net6.0+, SocketsHttpHandler injects the current Activity into request headers using the
-        // ActivityHeadersPropagator. This overwrites propagation headers already set by our HttpClient
-        // instrumentation. Disable this by setting a no-output propagator to preserve Datadog trace context.
+        // On net6.0+, SocketsHttpHandler builds an internal handler chain that includes DiagnosticsHandler
+        // when ActivityHeadersPropagator is non-null. DiagnosticsHandler creates Activities and fires
+        // DiagnosticSource events, which allows OpenTelemetry SDK (if present) to inject its own trace
+        // context headers, overwriting headers already set by our HttpClient instrumentation.
+        // Setting the propagator to null prevents DiagnosticsHandler from being added to the chain
+        // entirely, ensuring Datadog trace context is preserved.
         if (returnValue is System.Net.Http.SocketsHttpHandler handler)
         {
-            handler.ActivityHeadersPropagator = DistributedContextPropagator.CreateNoOutputPropagator();
+            handler.ActivityHeadersPropagator = null;
         }
 
         return new CallTargetReturn<TReturn>(returnValue);

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OcelotDistributedTracingTests.cs

@@ -24,6 +24,7 @@ public OcelotDistributedTracingTests(ITestOutputHelper output)
             : base("Ocelot.DistributedTracing", output)
         {
             SetServiceVersion(ServiceVersion);
+            SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "true");
         }
 
         public override Result ValidateIntegrationSpan(MockSpan span, string metadataSchemaVersion) => Result.DefaultSuccess;

tracer/test/snapshots/OcelotDistributedTracingTests.verified.txt

@@ -9,7 +9,7 @@
     Tags: {
       component: HttpMessageHandler,
       env: integration_tests,
-      http-client-handler-type: System.Net.Http.HttpClientHandler,
+      http-client-handler-type: System.Net.Http.SocketsHttpHandler,
       http.method: GET,
       http.status_code: 200,
       http.url: http://localhost:00000/proxy,
@@ -39,15 +39,21 @@
       env: integration_tests,
       http.method: GET,
       http.request.headers.host: localhost:00000,
+      http.request.method: GET,
       http.status_code: 200,
       http.url: http://localhost:00000/proxy,
       language: dotnet,
+      network.protocol.version: 1.1,
       runtime-id: Guid_1,
+      server.address: 127.0.0.1,
       span.kind: server,
+      url.path: /proxy,
+      url.scheme: http,
       version: 1.0.0
     },
     Metrics: {
       process_id: 0,
+      server.port: 55635.0,
       _dd.top_level: 1.0,
       _dd.tracer_kr: 1.0,
       _sampling_priority_v1: 1.0
@@ -93,16 +99,22 @@
       env: integration_tests,
       http.method: GET,
       http.request.headers.host: localhost:00000,
+      http.request.method: GET,
       http.route: /,
       http.status_code: 200,
       http.url: http://localhost:00000/,
       language: dotnet,
+      network.protocol.version: 1.1,
       runtime-id: Guid_1,
+      server.address: 127.0.0.1,
       span.kind: server,
+      url.path: /,
+      url.scheme: http,
       version: 1.0.0
     },
     Metrics: {
       process_id: 0,
+      server.port: 55635.0,
       _dd.top_level: 1.0,
       _dd.tracer_kr: 1.0,
       _sampling_priority_v1: 1.0

tracer/test/test-applications/integrations/Samples.Ocelot.DistributedTracing/Properties/launchSettings.json

@@ -10,7 +10,9 @@
           "CORECLR_PROFILER": "{846F5F1C-F9AE-4B07-969E-05C26BC060D8}",
           "CORECLR_PROFILER_PATH": "$(SolutionDir)shared\\bin\\monitoring-home\\win-x64\\Datadog.Trace.ClrProfiler.Native.dll",
 
-          "DD_DOTNET_TRACER_HOME": "$(SolutionDir)shared\\bin\\monitoring-home"
+          "DD_DOTNET_TRACER_HOME": "$(SolutionDir)shared\\bin\\monitoring-home",
+          "DD_TRACE_DEBUG": "true",
+          "DD_TRACE_OTEL_ENABLED": "true"
         },
         "applicationUrl": "http://localhost:5472/",
         "nativeDebugging": true

tracer/test/test-applications/integrations/Samples.Ocelot.DistributedTracing/Samples.Ocelot.DistributedTracing.csproj

@@ -14,6 +14,10 @@
 
   <ItemGroup>
     <PackageReference Include="Ocelot" Version="$(ApiVersion)" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.11.2" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.11.2" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.11.1" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.11.1" />
   </ItemGroup>
 
 </Project>

tracer/test/test-applications/integrations/Samples.Ocelot.DistributedTracing/Startup.cs

@@ -6,6 +6,8 @@
 using Microsoft.Extensions.Hosting;
 using Ocelot.DependencyInjection;
 using Ocelot.Middleware;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 using System.Collections.Generic;
 
 namespace Samples.Ocelot.DistributedTracing
@@ -14,6 +16,24 @@ public class Startup
     {
         public void ConfigureServices(IServiceCollection services)
         {
+            // Simulate a customer environment where the OpenTelemetry SDK is configured.
+            // The OTel SDK replaces DistributedContextPropagator.Current with its own propagator,
+            // which causes SocketsHttpHandler's internal DiagnosticsHandler to overwrite Datadog
+            // trace context headers on forwarded requests.
+            services.AddOpenTelemetry()
+                .ConfigureResource(r => r.AddService("ocelot-sample"))
+                .WithTracing(builder =>
+                {
+                    builder
+                        .AddAspNetCoreInstrumentation()
+                        .AddHttpClientInstrumentation()
+                        .AddOtlpExporter(o =>
+                        {
+                            // Use a non-routable endpoint so exports silently fail
+                            o.Endpoint = new System.Uri("http://192.0.2.1:4317");
+                        });
+                });
+
             // Create a minimal in-memory configuration for Ocelot startup
             // The actual routes will be configured at runtime by the Worker
             var ocelotConfig = new Dictionary<string, string>

tracer/test/test-applications/integrations/Samples.Ocelot.DistributedTracing/Worker.cs

@@ -60,8 +60,13 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                 // Update Ocelot's internal configuration to point the downstream route back to this application
                 await UpdateOcelotInternalConfig(serviceScope.ServiceProvider, address);
 
-                // Send a request through the proxy
-                using var client = new HttpClient();
+                // Send a request through the proxy.
+                // Disable ActivityHeadersPropagator on the Worker's HttpClient to prevent the OTel SDK
+                // from overwriting Datadog's trace context on the initial request. In a real scenario,
+                // the initial request comes from an external caller (e.g. RUM, Nginx) that isn't in
+                // this process, so this propagation conflict wouldn't occur.
+                var handler = new SocketsHttpHandler { ActivityHeadersPropagator = null };
+                using var client = new HttpClient(handler);
 
                 _logger.LogInformation("Sending request to self via Ocelot proxy");
                 var response = await client.GetAsync($"{address}/proxy", stoppingToken);