Distributed tracing in networking

docs/core/diagnostics/distributed-tracing-builtin-activities.md

@@ -11,6 +11,9 @@ This is a reference for distributed tracing [activities](xref:System.Diagnostics
 
 ## System.Net activities
 
+> [!TIP]
+> For a comprehensive guide about collecting and reporting `System.Net` traces, see [Networking distributed traces in .NET](../../fundamentals/networking/telemetry/metrics.md).
+
 ### HTTP client request
 
 <xref:System.Net.Http.SocketsHttpHandler> and <xref:System.Net.Http.HttpClientHandler> report the HTTP client request activity following the recommendations defined in OpenTelemetry [HTTP Client Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/http/http-spans/#http-client). It describes the HTTP request sent by <xref:System.Net.Http.HttpClient>'s send overloads during the time interval the underlying handler completes the request. Completing the request includes the time up to reading response headers from the network stream. It doesn't include the time spent reading the response body. <xref:System.Net.Http.SocketsHttpHandler> may retry requests, for example, on connection failures or HTTP version downgrades. Retries are not reported as separate *HTTP client request* activities.

docs/core/diagnostics/snippets/OTel-Prometheus-Grafana-Jaeger/csharp/Program.cs

@@ -41,6 +41,9 @@
     // Metrics provides by ASP.NET Core in .NET 8
     .AddMeter("Microsoft.AspNetCore.Hosting")
     .AddMeter("Microsoft.AspNetCore.Server.Kestrel")
+    // Metrics provided by System.Net libraries
+    .AddMeter("System.Net.Http")
+    .AddMeter("System.Net.NameResolution")
     .AddPrometheusExporter());
 
 // Add Tracing for ASP.NET Core and our custom ActivitySource and export to Jaeger
@@ -88,6 +91,24 @@
 app.MapGet("/NestedGreeting", SendNestedGreeting);
 //</Snippet_MapNested>
 
+//<Snippet_ClientStress>
+app.MapGet("/ClientStress", async Task<string> (ILogger<Program> logger, HttpClient client) =>
+{
+    string[] uris = ["http://example.com", "http://httpbin.org/get", "https://example.com", "https://httpbin.org/get"];
+    await Parallel.ForAsync(0, 50, async (_, ct) =>
+    {
+        string uri = uris[Random.Shared.Next(uris.Length)];
+
+        try
+        {
+            await client.GetAsync(uri, ct);
+            logger.LogInformation($"{uri} - done.");
+        }
+        catch { logger.LogInformation($"{uri} - failed."); }
+    });
+    return "Sent 50 requests to example.com and httpbin.org.";
+});
+//</Snippet_ClientStress>
 
 #if !AZURE_MONITOR
 //<Snippet_Prometheus>
@@ -99,7 +120,7 @@
 app.Run();
 
 //<Snippet_SendGreeting>
-async Task<String> SendGreeting(ILogger<Program> logger)
+async Task<string> SendGreeting(ILogger<Program> logger)
 {
     // Create a new Activity scoped to the method
     using var activity = greeterActivitySource.StartActivity("GreeterActivity");

docs/fundamentals/networking/telemetry/overview.md

@@ -8,9 +8,10 @@ ms.date: 10/18/2022
 
 # Networking telemetry in .NET
 
-The .NET networking stack is instrumented at various layers. .NET gives you the option to collect accurate timings throughout the lifetime of an HTTP request using metrics, event counters, and events.
+The .NET networking stack is instrumented at various layers. .NET gives you the option to collect accurate timings throughout the lifetime of an HTTP request using metrics, distributed tracing, event counters, and events.
 
-- **[Networking metrics](metrics.md)**: Starting with .NET 8, the HTTP and the name resolution (DNS) components are instrumented using the modern [System.Diagnostics.Metrics API](../../../core/diagnostics/metrics.md). These metrics were designed in cooperation with [OpenTelemetry](https://opentelemetry.io/).
+- **[Networking metrics](metrics.md)**: Starting with .NET 8, the HTTP and the name resolution (DNS) components are instrumented using the modern [System.Diagnostics.Metrics API](../../../core/diagnostics/metrics.md). These metrics were designed in cooperation with [OpenTelemetry](https://opentelemetry.io/) and can be exported to various monitoring tools.
+- **[Distributed tracing](tracing.md)**: `HttpClient` is instrumented to emit [distributed tracing](../../../core/diagnostics/distributed-tracing.md) activities (a.k.a., spans).
 - **[Networking events](events.md)**: Events provide debug and trace information with accurate timestamps.
 - **[Networking event counters](event-counters.md)**: All networking components are instrumented to publish real-time performance metrics using the EventCounters API.
 

docs/fundamentals/networking/telemetry/snippets/metrics/Program.cs

@@ -17,38 +17,15 @@
     await Parallel.ForAsync(0, Random.Shared.Next(20), async (_, ct) =>
     {
         string uri = uris[Random.Shared.Next(uris.Length)];
-        byte[] bytes = await client.GetByteArrayAsync(uri, ct);
-        await Console.Out.WriteLineAsync($"{uri} - received {bytes.Length} bytes.");
+        try
+        {
+            byte[] bytes = await client.GetByteArrayAsync(uri, ct);
+            await Console.Out.WriteLineAsync($"{uri} - received {bytes.Length} bytes.");
+        }
+        catch { await Console.Out.WriteLineAsync($"{uri} - failed."); }
     });
 }
 // </snippet_ExampleApp>
-#elif snippet_PrometheusExporter
-// <snippet_PrometheusExporter>
-using OpenTelemetry.Metrics;
-using OpenTelemetry;
-using System.Net;
-
-using MeterProvider meterProvider = Sdk.CreateMeterProviderBuilder()
-    .AddMeter("System.Net.Http", "System.Net.NameResolution")
-    .AddPrometheusHttpListener(options => options.UriPrefixes = new string[] { "http://localhost:9184/" })
-    .Build();
-
-string[] uris = ["http://example.com", "http://httpbin.org/get", "https://example.com", "https://httpbin.org/get"];
-using HttpClient client = new()
-{
-    DefaultRequestVersion = HttpVersion.Version20
-};
-
-while (!Console.KeyAvailable)
-{
-    await Parallel.ForAsync(0, Random.Shared.Next(20), async (_, ct) =>
-    {
-        string uri = uris[Random.Shared.Next(uris.Length)];
-        byte[] bytes = await client.GetByteArrayAsync(uri, ct);
-        await Console.Out.WriteLineAsync($"{uri} - received {bytes.Length} bytes.");
-    });
-}
-// </snippet_PrometheusExporter>
 #elif snippet_Enrichment
 // <snippet_Enrichment>
 using System.Net.Http.Metrics;

docs/fundamentals/networking/telemetry/snippets/tracing/ConnectionTracingDemo.ServiceDefaults/ConnectionTracingDemo.ServiceDefaults.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <IsAspireSharedProject>true</IsAspireSharedProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+
+    <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.10.0" />
+    <PackageReference Include="Microsoft.Extensions.ServiceDiscovery" Version="9.0.0-rc.1.24511.1" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
+  </ItemGroup>
+
+</Project>

docs/fundamentals/networking/telemetry/snippets/tracing/ConnectionTracingDemo.ServiceDefaults/Extensions.cs

@@ -0,0 +1,125 @@
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Diagnostics.HealthChecks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.ServiceDiscovery;
+using OpenTelemetry;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Trace;
+
+namespace Microsoft.Extensions.Hosting;
+
+// Adds common .NET Aspire services: service discovery, resilience, health checks, and OpenTelemetry.
+// This project should be referenced by each service project in your solution.
+// To learn more about using this project, see https://aka.ms/dotnet/aspire/service-defaults
+public static class Extensions
+{
+    public static TBuilder AddServiceDefaults<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
+    {
+        builder.ConfigureOpenTelemetry();
+
+        builder.AddDefaultHealthChecks();
+
+        builder.Services.AddServiceDiscovery();
+
+        builder.Services.ConfigureHttpClientDefaults(http =>
+        {
+            // Turn on resilience by default
+            http.AddStandardResilienceHandler();
+
+            // Turn on service discovery by default
+            http.AddServiceDiscovery();
+        });
+
+        // Uncomment the following to restrict the allowed schemes for service discovery.
+        // builder.Services.Configure<ServiceDiscoveryOptions>(options =>
+        // {
+        //     options.AllowedSchemes = ["https"];
+        // });
+
+        return builder;
+    }
+
+    public static TBuilder ConfigureOpenTelemetry<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
+    {
+        builder.Logging.AddOpenTelemetry(logging =>
+        {
+            logging.IncludeFormattedMessage = true;
+            logging.IncludeScopes = true;
+        });
+
+        builder.Services.AddOpenTelemetry()
+            // <snippet_Metrics>
+            .WithMetrics(metrics =>
+            {
+                metrics.AddAspNetCoreInstrumentation()
+                    .AddHttpClientInstrumentation()
+                    .AddRuntimeInstrumentation();
+            })
+            // </snippet_Metrics>
+            // <snippet_ConnectionTracing>
+            .WithTracing(tracing =>
+            {
+                tracing.AddAspNetCoreInstrumentation()
+                    // Instead of using .AddHttpClientInstrumentation()
+                    // .NET 9 allows to add the ActivitySources directly.
+                    .AddSource("System.Net.Http")
+                    // Add the experimental connection tracking ActivitySources using a wildcard.
+                    .AddSource("Experimental.System.Net.*");
+            });
+            // </snippet_ConnectionTracing>
+
+        builder.AddOpenTelemetryExporters();
+
+        return builder;
+    }
+
+
+    private static TBuilder AddOpenTelemetryExporters<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
+    {
+        var useOtlpExporter = !string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]);
+
+        if (useOtlpExporter)
+        {
+            builder.Services.AddOpenTelemetry().UseOtlpExporter();
+        }
+
+        // Uncomment the following lines to enable the Azure Monitor exporter (requires the Azure.Monitor.OpenTelemetry.AspNetCore package)
+        //if (!string.IsNullOrEmpty(builder.Configuration["APPLICATIONINSIGHTS_CONNECTION_STRING"]))
+        //{
+        //    builder.Services.AddOpenTelemetry()
+        //       .UseAzureMonitor();
+        //}
+
+        return builder;
+    }
+
+    public static TBuilder AddDefaultHealthChecks<TBuilder>(this TBuilder builder) where TBuilder : IHostApplicationBuilder
+    {
+        builder.Services.AddHealthChecks()
+            // Add a default liveness check to ensure app is responsive
+            .AddCheck("self", () => HealthCheckResult.Healthy(), ["live"]);
+
+        return builder;
+    }
+
+    public static WebApplication MapDefaultEndpoints(this WebApplication app)
+    {
+        // Adding health checks endpoints to applications in non-development environments has security implications.
+        // See https://aka.ms/dotnet/aspire/healthchecks for details before enabling these endpoints in non-development environments.
+        if (app.Environment.IsDevelopment())
+        {
+            // All health checks must pass for app to be considered ready to accept traffic after starting
+            app.MapHealthChecks("/health");
+
+            // Only health checks tagged with the "live" tag must pass for app to be considered alive
+            app.MapHealthChecks("/alive", new HealthCheckOptions
+            {
+                Predicate = r => r.Tags.Contains("live")
+            });
+        }
+
+        return app;
+    }
+}