Set correct total limits on OTel layer

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/arch/destination/SessionSpanWriter.kt

@@ -12,20 +12,20 @@ interface SessionSpanWriter {
      * Add a span event for the given [schemaType] to the session span. If [spanStartTimeMs] is null, the
      * current time will be used. Returns true if the event was added, otherwise false.
      */
-    fun addEvent(schemaType: SchemaType, startTimeMs: Long): Boolean
+    fun addSessionEvent(schemaType: SchemaType, startTimeMs: Long): Boolean
 
     /**
      * Remove all span events with the given [EmbType].
      */
-    fun removeEvents(type: EmbType)
+    fun removeSessionEvents(type: EmbType)
 
     /**
      * Add the given key-value pair as an Attribute to the session span
      */
-    fun addSystemAttribute(attribute: SpanAttributeData)
+    fun addSessionAttribute(attribute: SpanAttributeData)
 
     /**
      * Remove the attribute with the given key
      */
-    fun removeSystemAttribute(key: String)
+    fun removeSessionAttribute(key: String)
 }

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/capture/session/EmbraceSessionProperties.kt

@@ -56,7 +56,7 @@ internal class EmbraceSessionProperties(
                 }
                 temporary[sanitizedKey] = sanitizedValue
             }
-            writer.addSystemAttribute(
+            writer.addSessionAttribute(
                 SpanAttributeData(
                     sanitizedKey.toSessionPropertyAttributeName(),
                     sanitizedValue
@@ -79,7 +79,7 @@ internal class EmbraceSessionProperties(
                 preferencesService.permanentSessionProperties = permanentProperties()
                 existed = true
             }
-            writer.removeSystemAttribute(sanitizedKey.toSessionPropertyAttributeName())
+            writer.removeSessionAttribute(sanitizedKey.toSessionPropertyAttributeName())
             return existed
         }
     }
@@ -98,7 +98,7 @@ internal class EmbraceSessionProperties(
 
     fun addPermPropsToSessionSpan() {
         permanentProperties().entries.forEach {
-            writer.addSystemAttribute(
+            writer.addSessionAttribute(
                 SpanAttributeData(
                     it.key.toSessionPropertyAttributeName(),
                     it.value

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/opentelemetry/OpenTelemetrySdk.kt

@@ -3,6 +3,9 @@ package io.embrace.android.embracesdk.internal.opentelemetry
 import io.embrace.android.embracesdk.internal.Systrace
 import io.embrace.android.embracesdk.internal.config.instrumented.InstrumentedConfigImpl
 import io.embrace.android.embracesdk.internal.config.instrumented.schema.OtelLimitsConfig
+import io.embrace.android.embracesdk.internal.spans.getMaxTotalAttributeCount
+import io.embrace.android.embracesdk.internal.spans.getMaxTotalEventCount
+import io.embrace.android.embracesdk.internal.spans.getMaxTotalLinkCount
 import io.opentelemetry.api.logs.Logger
 import io.opentelemetry.api.trace.Tracer
 import io.opentelemetry.sdk.OpenTelemetrySdk
@@ -38,6 +41,7 @@ internal class OpenTelemetrySdk(
                         .toBuilder()
                         .setMaxNumberOfEvents(limits.getMaxTotalEventCount())
                         .setMaxNumberOfAttributes(limits.getMaxTotalAttributeCount())
+                        .setMaxNumberOfLinks(limits.getMaxTotalLinkCount())
                         .build()
                 )
                 .setClock(openTelemetryClock)

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/payload/SpanMapper.kt

@@ -45,7 +45,7 @@
 fun List<Attribute>.toOldPayload(): Map<String, String> =
     associate { Pair(it.key ?: "", it.data ?: "") }.filterKeys { it.isNotBlank() }
 
-fun EmbraceLinkData.toPayload() = Link(spanContext.spanId, attributes?.toNewPayload())
+fun EmbraceLinkData.toPayload() = Link(spanContext.spanId, attributes.toNewPayload())
 
 fun Span.toOldPayload(): EmbraceSpanData {
     return EmbraceSpanData(

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/session/orchestrator/SessionOrchestratorImpl.kt

@@ -247,7 +247,7 @@ internal class SessionOrchestratorImpl(
     private fun updatePeriodicCacheAttrs() {
         val now = clock.now().millisToNanos()
         val attr = SpanAttributeData(embHeartbeatTimeUnixNano.name, now.toString())
-        sessionSpanWriter.addSystemAttribute(attr)
-        sessionSpanWriter.addSystemAttribute(SpanAttributeData(embTerminated.name, true.toString()))
+        sessionSpanWriter.addSessionAttribute(attr)
+        sessionSpanWriter.addSessionAttribute(SpanAttributeData(embTerminated.name, true.toString()))
     }
 }

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/session/orchestrator/SessionSpanAttrPopulatorImpl.kt

@@ -29,39 +29,39 @@ internal class SessionSpanAttrPopulatorImpl(
 
     override fun populateSessionSpanStartAttrs(session: SessionZygote) {
         with(sessionSpanWriter) {
-            addSystemAttribute(SpanAttributeData(embColdStart.name, session.isColdStart.toString()))
-            addSystemAttribute(SpanAttributeData(embSessionNumber.name, session.number.toString()))
-            addSystemAttribute(SpanAttributeData(embState.name, session.appState.name.lowercase(Locale.US)))
-            addSystemAttribute(SpanAttributeData(embCleanExit.name, false.toString()))
-            addSystemAttribute(SpanAttributeData(embTerminated.name, true.toString()))
+            addSessionAttribute(SpanAttributeData(embColdStart.name, session.isColdStart.toString()))
+            addSessionAttribute(SpanAttributeData(embSessionNumber.name, session.number.toString()))
+            addSessionAttribute(SpanAttributeData(embState.name, session.appState.name.lowercase(Locale.US)))
+            addSessionAttribute(SpanAttributeData(embCleanExit.name, false.toString()))
+            addSessionAttribute(SpanAttributeData(embTerminated.name, true.toString()))
 
             session.startType.toString().lowercase(Locale.US).let {
-                addSystemAttribute(SpanAttributeData(embSessionStartType.name, it))
+                addSessionAttribute(SpanAttributeData(embSessionStartType.name, it))
             }
         }
     }
 
     override fun populateSessionSpanEndAttrs(endType: LifeEventType?, crashId: String?, coldStart: Boolean) {
         with(sessionSpanWriter) {
-            addSystemAttribute(SpanAttributeData(embCleanExit.name, true.toString()))
-            addSystemAttribute(SpanAttributeData(embTerminated.name, false.toString()))
+            addSessionAttribute(SpanAttributeData(embCleanExit.name, true.toString()))
+            addSessionAttribute(SpanAttributeData(embTerminated.name, false.toString()))
             crashId?.let {
-                addSystemAttribute(SpanAttributeData(embCrashId.name, crashId))
+                addSessionAttribute(SpanAttributeData(embCrashId.name, crashId))
             }
             endType?.toString()?.lowercase(Locale.US)?.let {
-                addSystemAttribute(SpanAttributeData(embSessionEndType.name, it))
+                addSessionAttribute(SpanAttributeData(embSessionEndType.name, it))
             }
             if (coldStart) {
                 startupService.getSdkStartupDuration()?.let { duration ->
-                    addSystemAttribute(SpanAttributeData(embSessionStartupDuration.name, duration.toString()))
+                    addSessionAttribute(SpanAttributeData(embSessionStartupDuration.name, duration.toString()))
                 }
             }
 
             val logCount = logService.getErrorLogsCount()
-            addSystemAttribute(SpanAttributeData(embErrorLogCount.name, logCount.toString()))
+            addSessionAttribute(SpanAttributeData(embErrorLogCount.name, logCount.toString()))
 
             metadataService.getDiskUsage()?.deviceDiskFree?.let { free ->
-                addSystemAttribute(SpanAttributeData(embFreeDiskBytes.name, free.toString()))
+                addSessionAttribute(SpanAttributeData(embFreeDiskBytes.name, free.toString()))
             }
         }
     }

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/spans/CurrentSessionSpanImpl.kt

@@ -137,7 +137,7 @@ internal class CurrentSessionSpanImpl(
         }
     }
 
-    override fun addEvent(schemaType: SchemaType, startTimeMs: Long): Boolean {
+    override fun addSessionEvent(schemaType: SchemaType, startTimeMs: Long): Boolean {
         val currentSession = sessionSpan.get() ?: return false
         return currentSession.addSystemEvent(
             schemaType.fixedObjectName.toEmbraceObjectName(),
@@ -146,17 +146,17 @@ internal class CurrentSessionSpanImpl(
         )
     }
 
-    override fun removeEvents(type: EmbType) {
+    override fun removeSessionEvents(type: EmbType) {
         val currentSession = sessionSpan.get() ?: return
         currentSession.removeSystemEvents(type)
     }
 
-    override fun addSystemAttribute(attribute: SpanAttributeData) {
+    override fun addSessionAttribute(attribute: SpanAttributeData) {
         val currentSession = sessionSpan.get() ?: return
         currentSession.addSystemAttribute(attribute.key, attribute.value)
     }
 
-    override fun removeSystemAttribute(key: String) {
+    override fun removeSessionAttribute(key: String) {
         val currentSession = sessionSpan.get() ?: return
         currentSession.removeSystemAttribute(key)
     }

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/spans/EmbraceExtensions.kt

@@ -104,3 +104,9 @@ fun StatusCode.toStatus(): Span.Status {
         StatusCode.ERROR -> io.embrace.android.embracesdk.internal.payload.Span.Status.ERROR
     }
 }
+
+fun OtelLimitsConfig.getMaxTotalAttributeCount() = getMaxSystemAttributeCount() + getMaxCustomAttributeCount()
+
+fun OtelLimitsConfig.getMaxTotalEventCount() = getMaxSystemEventCount() + getMaxCustomEventCount()
+
+fun OtelLimitsConfig.getMaxTotalLinkCount() = getMaxSystemLinkCount() + getMaxCustomLinkCount()

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/spans/EmbraceLinkData.kt

@@ -4,5 +4,5 @@ import io.opentelemetry.api.trace.SpanContext
 
 data class EmbraceLinkData(
     val spanContext: SpanContext,
-    val attributes: Map<String, String>?
+    val attributes: Map<String, String>
 )

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/spans/EmbraceSpanImpl.kt

@@ -66,11 +66,14 @@
         putAll(spanBuilder.getCustomAttributes())
     }
     private val systemLinks = ConcurrentLinkedQueue<EmbraceLinkData>()
+    private val customLinks = ConcurrentLinkedQueue<EmbraceLinkData>()
 
     // size for ConcurrentLinkedQueues is not a constant operation, so it could be subject to race conditions
     // do the bookkeeping separately so we don't have to worry about this
     private val systemEventCount = AtomicInteger(0)
     private val customEventCount = AtomicInteger(0)
+    private val systemLinkCount = AtomicInteger(0)
+    private val customLinkCount = AtomicInteger(0)
 
     override val parent: EmbraceSpan? = spanBuilder.getParentSpan()
 
@@ -153,19 +156,8 @@
         return successful
     }
 
-    private fun populateLinks(spanToStop: io.opentelemetry.api.trace.Span) {
-        systemLinks.forEach {
-            val linkAttributes = if (it.attributes != null) {
-                Attributes.builder().fromMap(attributes = it.attributes, false).build()
-            } else {
-                Attributes.empty()
-            }
-            spanToStop.addLink(it.spanContext, linkAttributes)
-        }
-    }
-
     override fun addEvent(name: String, timestampMs: Long?, attributes: Map<String, String>?): Boolean =
-        recordEvent(customEvents, customEventCount, limits.getMaxCustomEventCount()) {
+        addObject(customEvents, customEventCount, limits.getMaxCustomEventCount()) {
             EmbraceSpanEvent.create(
                 name = name,
                 timestampMs = timestampMs?.normalizeTimestampAsMillis() ?: openTelemetryClock.now().nanosToMillis(),
@@ -174,7 +166,7 @@
         }
 
     override fun recordException(exception: Throwable, attributes: Map<String, String>?): Boolean =
-        recordEvent(customEvents, customEventCount, limits.getMaxCustomEventCount()) {
+        addObject(customEvents, customEventCount, limits.getMaxCustomEventCount()) {
             val eventAttributes = mutableMapOf<String, String>()
             if (attributes != null) {
                 eventAttributes.putAll(attributes)
@@ -198,7 +190,7 @@
         }
 
     override fun addSystemEvent(name: String, timestampMs: Long?, attributes: Map<String, String>?): Boolean =
-        recordEvent(systemEvents, systemEventCount, limits.getMaxTotalEventCount()) {
+        addObject(systemEvents, systemEventCount, limits.getMaxSystemEventCount()) {
             EmbraceSpanEvent.create(
                 name = name,
                 timestampMs = timestampMs?.normalizeTimestampAsMillis() ?: openTelemetryClock.now().nanosToMillis(),
@@ -261,24 +253,21 @@
         return false
     }
 
-    override fun addLink(linkedSpanContext: SpanContext, attributes: Map<String, String>?): Boolean {
-        if (systemLinks.size < limits.getMaxTotalLinkCount()) {
-            synchronized(systemLinks) {
-                if (systemLinks.size < limits.getMaxTotalLinkCount()) {
-                    systemLinks.add(EmbraceLinkData(linkedSpanContext, attributes))
-                    spanRepository.notifySpanUpdate()
-                    return true
-                }
-            }
+    override fun addSystemLink(linkedSpanContext: SpanContext, attributes: Map<String, String>): Boolean =
+        addObject(systemLinks, systemLinkCount, limits.getMaxSystemLinkCount()) {
+            EmbraceLinkData(linkedSpanContext, attributes)
         }
 
-        return false
-    }
+    override fun addLink(linkedSpanContext: SpanContext, attributes: Map<String, String>?): Boolean =
+        addObject(customLinks, customLinkCount, limits.getMaxCustomLinkCount()) {
+            EmbraceLinkData(linkedSpanContext, attributes ?: emptyMap())
+        }
 
     override fun asNewContext(): Context? = startedSpan.get()?.run { spanBuilder.parentContext.with(this) }
 
     override fun snapshot(): Span? {
         val redactedCustomEvents = customEvents.map { it.copy(attributes = it.attributes.redactIfSensitive()) }
+        val redactedCustomLinks = customLinks.map { it.copy(attributes = it.attributes.redactIfSensitive()) }
         return if (canSnapshot()) {
             Span(
                 traceId = traceId,
@@ -290,7 +279,7 @@
                 status = status,
                 events = systemEvents.map(EmbraceSpanEvent::toNewPayload) + redactedCustomEvents.map(EmbraceSpanEvent::toNewPayload),
                 attributes = getAttributesPayload(),
-                links = systemLinks.toList().map { it.toPayload() }
+                links = (systemLinks + redactedCustomLinks).map(EmbraceLinkData::toPayload)
             )
         } else {
             null
@@ -321,17 +310,17 @@
 
     private fun canSnapshot(): Boolean = spanId != null && spanStartTimeMs != null
 
-    private fun recordEvent(
-        events: Queue<EmbraceSpanEvent>,
+    private fun <T> addObject(
+        queue: Queue<T>,
         count: AtomicInteger,
         max: Int,
-        eventSupplier: () -> EmbraceSpanEvent?,
+        objectSupplier: () -> T?,
     ): Boolean {
         if (count.get() < max) {
             synchronized(count) {
                 if (count.get() < max && isRecording) {
-                    eventSupplier()?.apply {
-                        events.add(this)
+                    objectSupplier()?.apply {
+                        queue.add(this)
                         count.incrementAndGet()
                         spanRepository.notifySpanUpdate()
                         return true
@@ -384,4 +373,17 @@
             )
         }
     }
+
+    private fun populateLinks(spanToStop: io.opentelemetry.api.trace.Span) {
+        val redactedCustomLinks = customLinks.map { it.copy(attributes = it.attributes.redactIfSensitive()) }
+
+        (systemLinks + redactedCustomLinks).forEach {
+            val linkAttributes = if (it.attributes.isNotEmpty()) {
+                Attributes.builder().fromMap(attributes = it.attributes, false).build()
+            } else {
+                Attributes.empty()
+            }
+            spanToStop.addLink(it.spanContext, linkAttributes)
+        }
+    }
 }

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/spans/PersistableEmbraceSpan.kt

@@ -5,6 +5,7 @@
 import io.embrace.android.embracesdk.internal.payload.Span
 import io.embrace.android.embracesdk.spans.EmbraceSpan
 import io.opentelemetry.api.common.AttributeKey
+import io.opentelemetry.api.trace.SpanContext
 import io.opentelemetry.api.trace.StatusCode
 import io.opentelemetry.context.Context
 import io.opentelemetry.context.ContextKey
@@ -72,6 +73,14 @@
 
     fun getStartTimeMs(): Long?
 
+    /**
+     * Add a system link to the span that will subjected to a different maximum than typical links.
+     */
+    fun addSystemLink(
+        linkedSpanContext: SpanContext,
+        attributes: Map<String, String> = emptyMap(),
+    ): Boolean
+
     override fun storeInContext(context: Context): Context = context.with(embraceSpanContextKey, this)
 }