Set DD_TRACE_OTEL_ENABLED=true by default for all integration tests (#8370)

## Summary of changes

- Set `DD_TRACE_OTEL_ENABLED=true` by default in integration tests
- Add additional excludes for activity handlers with equivalent custom
instrumentation

## Reason for change

We had an escalation recently, which highlighted that we were missing
some entries in the `IgnoreActivityHandler`. To avoid hitting similar
issues in the future, we can set `DD_TRACE_OTEL_ENABLED=true` by
default, so we know as soon as a new `ActivitySource` lights up.

## Implementation details

A _lot_ of trial and error here, mostly setting
`DD_TRACE_OTEL_ENABLED=true` in the `TracingIntegrationTest` base class
and seeing what breaks 😅 That pointed to a variety of extra handlers and
differences in behaviour:

- `Couchbase.DotnetSdk.OpenTelemetryRequestTracer`, I don't think we
actually need this one strictly, but it showed up while I was trying to
fix persistent issues with Couchbase3, so I think it makes sense to
exclude it
- The _real_ issue I had is that some early versions of Couchbase (3.0.0
- 3.2.0) create activities using `new Activity()`, which means there's
_no_ `ActivitySource` associated, and so we have no way to filter them 💀
- Rather than fight with that, and because those versions are
deprecated, just disabled OTel integration for these specific tested
versions
- `connector-net` - this is the ActivitySource for `MySql.Data` (we
already excluded the one for `MySqlConnector`)
- `RabbitMQ.Client.*` - these were the ones that caused the original
issue, and were breaking DSM
- `Experimental.System.Net.Security` - this one came in .NET 9, and was
causing extra spans in gRPC and Yarp tests
- `Grpc.Net.Client` - The gRPC client has had an `ActivitySource` [for a
long time](grpc/grpc-dotnet#2244) 😅
- `Yarp.ReverseProxy` - ...[as has
Yarp](dotnet/yarp#2098)

In addition, there were some extra "fixes" to the tests required:

- For the `HttpMessageHandler` tests, where the integration is disabled,
we were previously verifying that W3C headers weren't injected, but they
_will_ be if OTel is enabled, so just relaxed the restrictions there.
- For `OpenTelemetrySdkTests.SubmitsOtlpLogs`, the data changes
depending on whether otel is enabled or not, so just reset it to the
default for simplicity rather than wrestle with it (I spent some time
ping-ponging snapshots before I gave up 😅)
- Updated the gRPC snapshots to add `grpc.method` and `grpc.status_code`
which are now added to the aspnetcore spans

## Test coverage

Hopefully self explanatory 😅 

## Other details

Fixes https://datadoghq.atlassian.net/browse/DSMS-138

Technically, this could be a breaking change for some people, so maybe
we should revisit making the ignore activity handler configurable?

Author: andrewlock

tracer/src/Datadog.Trace/Activity/Handlers/IgnoreActivityHandler.cs

@@ -19,17 +19,24 @@ internal sealed class IgnoreActivityHandler : IActivityHandler
         private static readonly string[] SourcesNames =
         {
             "Couchbase.DotnetSdk.RequestTracer",
+            "Couchbase.DotnetSdk.OpenTelemetryRequestTracer",
+            "Grpc.Net.Client",
             "HttpHandlerDiagnosticListener",
             "Microsoft.AspNetCore",
             "Microsoft.EntityFrameworkCore",
             "MongoDB.Driver",
             "MySqlConnector",
+            "connector-net",
             "Npgsql",
+            "RabbitMQ.Client.Publisher",
+            "RabbitMQ.Client.Subscriber",
             "System.Net.Http.Desktop",
             "SqlClientDiagnosticListener",
             "Experimental.System.Net.NameResolution",
             "Experimental.System.Net.Http.Connections",
+            "Experimental.System.Net.Security",
             "Experimental.System.Net.Sockets",
+            "Yarp.ReverseProxy",
         };
 
         public static bool ShouldIgnoreByOperationName(string? operationName)

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AspNetCore/AspNetCoreMvcTestBase.cs

@@ -61,9 +61,6 @@ protected AspNetCoreMvcTestBase(string sampleName, AspNetCoreTestFixture fixture
             SetEnvironmentVariable(ConfigurationKeys.FeatureFlags.RouteTemplateResourceNamesEnabled, (flags == AspNetCoreFeatureFlags.RouteTemplateResourceNames).ToString());
             SetEnvironmentVariable(ConfigurationKeys.FeatureFlags.SingleSpanAspNetCoreEnabled, (flags == AspNetCoreFeatureFlags.SingleSpan).ToString());
 
-            // Enable OpenTelemetry interop to test that the OTel Baggage API integration works correctly.
-            SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "true");
-
             Fixture = fixture;
             Fixture.SetOutput(output);
         }

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Azure/AzureServiceBusTests.cs

@@ -48,7 +48,6 @@ public static IEnumerable<object[]> GetEnabledConfig()
         public async Task SubmitsTraces(string packageVersion, string metadataSchemaVersion)
         {
             SetEnvironmentVariable("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", metadataSchemaVersion);
-            SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "true");
 
             // If you want to use a custom connection string, set it here
             // SetEnvironmentVariable("ASB_CONNECTION_STRING", null);

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Azure/DataStreamsMonitoringAzureServiceBusTests.cs

@@ -38,7 +38,6 @@ public static IEnumerable<object[]> GetPackageVersions()
     public async Task HandleProduceAndConsume(string packageVersion)
     {
         SetEnvironmentVariable(ConfigurationKeys.DataStreamsMonitoring.Enabled, "1");
-        SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "true");
 
         // If you want to use a custom connection string, set it here
         // SetEnvironmentVariable("ASB_CONNECTION_STRING", null);
@@ -67,7 +66,6 @@ await Verifier.Verify(PayloadsToPoints(agent.DataStreams), settings)
     public async Task ValidateSpanTags(string packageVersion)
     {
         SetEnvironmentVariable(ConfigurationKeys.DataStreamsMonitoring.Enabled, "1");
-        SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "true");
 
         // If you want to use a custom connection string, set it here
         // SetEnvironmentVariable("ASB_CONNECTION_STRING", null);

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Couchbase3Tests.cs

@@ -42,6 +42,15 @@ public static IEnumerable<object[]> GetEnabledConfig()
         public async Task SubmitTraces(string packageVersion, string metadataSchemaVersion)
         {
             SetEnvironmentVariable("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", metadataSchemaVersion);
+
+            if (GetPackageVersion(packageVersion) < new Version(3, 2, 0))
+            {
+                // Early versions of tracing just called new Activity(), which results in activities with no source, so
+                // our IgnoreActivityHandler fails to detect it: https://github.com/couchbase/couchbase-net-client/blob/3.1.0/src/Couchbase/Core/Diagnostics/Tracing/ActivityRequestTracer.cs#L24
+                // for simplicity, just disable OTEL integration for these cases
+                SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "false");
+            }
+
             var isExternalSpan = metadataSchemaVersion == "v0";
             var clientSpanServiceName = isExternalSpan ? $"{EnvironmentHelper.FullSampleName}-couchbase" : EnvironmentHelper.FullSampleName;
 
@@ -50,8 +59,8 @@ public async Task SubmitTraces(string packageVersion, string metadataSchemaVersi
             using (await RunSampleAndWaitForExit(agent, packageVersion: packageVersion))
             {
                 var spans = (await agent.WaitForSpansAsync(10, 500))
-                                 .Where(s => s.Type == "db")
-                                 .ToList();
+                           .Where(s => s.Type == "db")
+                           .ToList();
 
                 ValidateIntegrationSpans(spans, metadataSchemaVersion, expectedServiceName: clientSpanServiceName, isExternalSpan);
 
@@ -91,7 +100,7 @@ await VerifyHelper.VerifySpans(spans, settings)
 
         private static string GetVersionSuffix(string packageVersion)
         {
-            var version = new Version(string.IsNullOrEmpty(packageVersion) ? "3.4.1" : packageVersion); // default version in csproj
+            var version = GetPackageVersion(packageVersion);
             if (version < new Version("3.1.2"))
             {
                 return "_3_0";
@@ -119,5 +128,8 @@ private static string GetVersionSuffix(string packageVersion)
 
             return string.Empty;
         }
+
+        private static Version GetPackageVersion(string packageVersion)
+            => new(string.IsNullOrEmpty(packageVersion) ? "3.4.1" : packageVersion); // default version in csproj
     }
 }

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/GrpcTests.cs

@@ -29,7 +29,6 @@ public class GrpcLegacyTests : GrpcTestsBase
         public GrpcLegacyTests(ITestOutputHelper output)
             : base("GrpcLegacy", output, usesAspNetCore: false)
         {
-            SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "true");
             // the sample uses protobuf, but we are only interested in testing the grpc instrumentation,
             // so we disable the proto one which would add unexpected tags to the spans.
             SetEnvironmentVariable("DD_TRACE_PROTOBUF_ENABLED", "false");
@@ -373,6 +372,7 @@ static void FixVerySlowServerSpans(IImmutableList<MockSpan> spans, HttpClientInt
                         {
                             // May be missing in some cases
                             span.Tags["http.status_code"] = "200";
+                            span.Tags["grpc.status_code"] = "4";
                         }
 
                         // there is a race between the server cancelling a deadline and the client cancelling it

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/HangfireTests.cs

@@ -34,9 +34,6 @@ public HangfireTests(ITestOutputHelper output)
     [Trait("RunOnWindows", "True")]
     public async Task SubmitsTraces()
     {
-        // needed for context propagation test
-        SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "true");
-
         using (var telemetry = this.ConfigureTelemetry())
         using (var agent = EnvironmentHelper.GetMockAgent())
         using (await RunSampleAndWaitForExit(agent))

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/HttpMessageHandlerTests.cs

@@ -212,8 +212,11 @@ public async Task TracingDisabled_DoesNotSubmitsTraces(
                     headers.Should().NotContainKey(HttpHeaderNames.PropagatedTags);
 
                     // W3C trace context headers
-                    headers.Should().NotContainKey(W3CTraceContextPropagator.TraceParentHeaderName);
-                    headers.Should().NotContainKey(W3CTraceContextPropagator.TraceStateHeaderName);
+                    // These are still injected when we have otel enabled, but they're not added by us
+                    // TODO: Verify whether the injected values are actually _correct_ - they _should_ be,
+                    // because our IgnoreActivityHandler should fix them
+                    // headers.Should().NotContainKey(W3CTraceContextPropagator.TraceParentHeaderName);
+                    // headers.Should().NotContainKey(W3CTraceContextPropagator.TraceStateHeaderName);
 
                     // B3 trace context headers
                     headers.Should().NotContainKey(B3SingleHeaderContextPropagator.B3);

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/MongoDbTests.cs

@@ -48,7 +48,6 @@ public static IEnumerable<object[]> GetEnabledConfig()
         public async Task SubmitsTraces(string packageVersion, string metadataSchemaVersion)
         {
             SetEnvironmentVariable("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", metadataSchemaVersion);
-            SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "true");
 
             var isExternalSpan = metadataSchemaVersion == "v0";
             var clientSpanServiceName = isExternalSpan ? $"{EnvironmentHelper.FullSampleName}-mongodb" : EnvironmentHelper.FullSampleName;

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/NetActivitySdkTests.cs

@@ -59,7 +59,6 @@ public NetActivitySdkTests(ITestOutputHelper output)
         [Trait("RunOnWindows", "True")]
         public async Task SubmitsTraces()
         {
-            SetEnvironmentVariable("DD_TRACE_OTEL_ENABLED", "true");
             SetEnvironmentVariable("DD_TRACE_DISABLED_ACTIVITY_SOURCES", "Disabled.By.ExactMatch,*.By.Glob*");
 
             using (var telemetry = this.ConfigureTelemetry())