Improve OTel bridge mapping of newer semcon for databases

Author: stevejgordon

src/Elastic.Apm/OpenTelemetry/OTelActivityMapper.cs

@@ -5,6 +5,7 @@
 
 #if NET || NETSTANDARD2_1
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
 using Elastic.Apm.Api;
@@ -19,15 +20,38 @@ namespace Elastic.Apm.OpenTelemetry
 	internal static class OTelActivityMapper
 	{
 		internal static readonly string[] ServerPortAttributeKeys =
-			[SemanticConventions.ServerPort, SemanticConventions.NetPeerPort];
+			[SemanticConventions.ServerPort, SemanticConventions.NetworkPeerPort, SemanticConventions.NetPeerPort];
 
 		internal static readonly string[] ServerAddressAttributeKeys =
-			[SemanticConventions.ServerAddress, SemanticConventions.NetPeerName, SemanticConventions.NetPeerIp];
+			[SemanticConventions.ServerAddress, SemanticConventions.NetworkPeerAddress, SemanticConventions.NetPeerName, SemanticConventions.NetPeerIp];
 
 		// Canonical URL/host-presence keys used both as "is this HTTP?" and for URL parsing.
 		internal static readonly string[] HttpAttributeKeys =
 			[SemanticConventions.UrlFull, SemanticConventions.HttpUrl];
 
+		internal static readonly string[] DbSystemAttributeKeys =
+			[SemanticConventions.DbSystemName, SemanticConventions.DbSystem];
+
+		internal static readonly string[] DbInstanceAttributeKeys =
+			[SemanticConventions.DbNamespace, SemanticConventions.DbName];
+
+		internal static readonly string[] DbQueryTextAttributeKeys =
+			[SemanticConventions.DbQueryText, SemanticConventions.DbStatement];
+
+		// Systems where db.namespace is the confirmed stable replacement for db.name and maps correctly
+		// to span.db.instance / service.target.name. Validated against each system's OTel spec page:
+		//   mongodb   — db.namespace = database name
+		//   mysql     — db.namespace = database name
+		//   cassandra — db.namespace = keyspace (the Cassandra equivalent of a database)
+		//   cosmosdb  — db.namespace = database name ("azure.cosmosdb" is normalized to "cosmosdb" before this check)
+		// Excluded systems and why:
+		//   elasticsearch — db.namespace = cluster name, not the index/alias being accessed
+		//   postgresql    — db.namespace = "{database}|{schema}", composite; changes service target grouping vs db.name
+		//   mssql         — db.namespace = "{instance}|{database}", composite; same concern as postgresql
+		// All other systems use db.name only.
+		internal static readonly HashSet<string> DbNamespaceAsInstanceSystems =
+			new(StringComparer.OrdinalIgnoreCase) { "mongodb", "mysql", "cassandra", "cosmosdb" };
+
 		internal static void UpdateOTelAttributes(Activity activity, OTel otel)
 		{
 			var i = 0;
@@ -65,14 +89,15 @@ internal static void InferTransactionType(Transaction transaction, Activity acti
 		internal static void InferSpanTypeAndSubType(Span span, Activity activity)
 		{
 			var peerPort = string.Empty;
-			var netName = string.Empty;
+			var peerAddress = string.Empty;
 
 			if (TryGetStringValue(activity, ServerPortAttributeKeys, out var netPortValue))
 				peerPort = netPortValue;
 
 			if (TryGetStringValue(activity, ServerAddressAttributeKeys, out var netNameValue))
-				netName = netNameValue;
+				peerAddress = netNameValue;
 
+			var netName = peerAddress;
 			if (netName.Length > 0 && peerPort.Length > 0)
 			{
 				netName += ':';
@@ -83,13 +108,37 @@ internal static void InferSpanTypeAndSubType(Span span, Activity activity)
 			string serviceTargetName = null;
 			string resource = null;
 
-			if (TryGetStringValue(activity, SemanticConventions.DbSystem, out var dbSystem))
+			// db.system.name is the current OTel convention; db.system is the older one still used by many libraries.
+			if (TryGetStringValue(activity, DbSystemAttributeKeys, out var dbSystem))
 			{
+				// Normalize OTel system names to the Elastic APM subtype values expected by APM server / Kibana.
+				// The raw OTel value is preserved in otel.attributes; this only affects ECS-mapped fields.
+				dbSystem = NormalizeDbSystem(dbSystem);
 				span.Type = ApiConstants.TypeDb;
 				span.Subtype = dbSystem;
+				span.Action = ApiConstants.ActionQuery;
 				serviceTargetType = span.Subtype;
-				serviceTargetName = TryGetStringValue(activity, SemanticConventions.DbName, out var dbName) ? dbName : null;
+
+				// db.namespace semantics vary by system — only use it for systems where it is a direct
+				// equivalent of the database name / instance concept (see DbNamespaceAsInstanceSystems).
+				// For others (e.g. elasticsearch = cluster name, postgresql = "{db}|{schema}") fall back
+				// to db.name only, to avoid incorrect service target grouping.
+				string dbInstance = null;
+				if (DbNamespaceAsInstanceSystems.Contains(dbSystem))
+					TryGetStringValue(activity, DbInstanceAttributeKeys, out dbInstance);
+				else
+					TryGetStringValue(activity, SemanticConventions.DbName, out dbInstance);
+				serviceTargetName = dbInstance;
 				resource = ToResourceName(span.Subtype, serviceTargetName);
+
+				// db.query.text is the current OTel convention; db.statement is the older one.
+				TryGetStringValue(activity, DbQueryTextAttributeKeys, out var dbStatement);
+				span.Context.Db = new Database
+				{
+					Type = dbSystem,
+					Instance = dbInstance,
+					Statement = dbStatement
+				};
 			}
 			else if (TryGetStringValue(activity, SemanticConventions.MessagingSystem, out var messagingSystem))
 			{
@@ -154,6 +203,13 @@ internal static void InferSpanTypeAndSubType(Span span, Activity activity)
 				span.Context.Destination ??= new Destination();
 				span.Context.Destination.Service = new Destination.DestinationService { Resource = resource };
 			}
+			if (peerAddress.Length > 0)
+			{
+				span.Context.Destination ??= new Destination();
+				span.Context.Destination.Address = peerAddress;
+				if (peerPort.Length > 0 && int.TryParse(peerPort, out var parsedPort))
+					span.Context.Destination.Port = parsedPort;
+			}
 		}
 
 		/// <summary>
@@ -237,6 +293,16 @@ private static string ParseNetName(string url)
 			}
 		}
 
+		/// <summary>
+		/// Maps OTel <c>db.system.name</c> values to the Elastic APM subtype string expected by APM server.
+		/// Only applied to ECS-mapped fields; raw OTel attributes are stored as-is.
+		/// </summary>
+		private static string NormalizeDbSystem(string dbSystem) =>
+			// "azure.cosmosdb" is the stable OTel value; APM server / Kibana expect "cosmosdb" (matching the native integration).
+			string.Equals(dbSystem, "azure.cosmosdb", StringComparison.OrdinalIgnoreCase)
+				? ApiConstants.SubTypeCosmosDb
+				: dbSystem;
+
 		private static string ToResourceName(string type, string name) =>
 			string.IsNullOrEmpty(name) ? type : $"{type}/{name}";
 	}

src/Elastic.Apm/OpenTelemetry/SemanticConventions.cs

@@ -4,7 +4,13 @@ internal static class SemanticConventions
 {
 	// DATABASE
 	public const string DbSystem = "db.system";
+	public const string DbSystemName = "db.system.name"; // newer OTel convention (replaces db.system)
 	public const string DbName = "db.name";
+	// db.namespace is the current OTel convention replacing db.name. The value is system-specific:
+	// MySQL/MongoDB: the database name. PostgreSQL: "{database}|{schema}". SQL Server: "{instance}|{database}". Redis: the numeric db index.
+	public const string DbNamespace = "db.namespace";
+	public const string DbStatement = "db.statement";
+	public const string DbQueryText = "db.query.text"; // newer OTel convention (replaces db.statement)
 
 	// HTTP
 	public const string HttpUrl = "http.url";
@@ -28,7 +34,11 @@ internal static class SemanticConventions
 	public const string ServerAddress = "server.address";
 	public const string ServerPort = "server.port";
 
-	// NET
+	// NETWORK (stable, current)
+	public const string NetworkPeerAddress = "network.peer.address";
+	public const string NetworkPeerPort = "network.peer.port";
+
+	// NET (legacy)
 	public const string NetPeerIp = "net.peer.ip";
 	public const string NetPeerName = "net.peer.name";
 	public const string NetPeerPort = "net.peer.port";