From d9fff336588136594247bfd6f693d5d17bab1779 Mon Sep 17 00:00:00 2001
From: Mike Terhar <mike@terhar.com>
Date: Thu, 10 Apr 2025 16:23:32 +0000
Subject: [PATCH 1/2] add a retry on span routing

---
 route/route.go | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/route/route.go b/route/route.go
index 0ca1895026..9144add352 100644
--- a/route/route.go
+++ b/route/route.go
@@ -661,9 +661,23 @@ func (r *Router) processEvent(ev *types.Event, reqID interface{}) error {
 		err = r.Collector.AddSpanFromPeer(span)
 	}
 	if err != nil {
-		r.Metrics.Increment(r.incomingOrPeer + "_router_dropped")
-		debugLog.Logf("Dropping span from batch, channel full")
-		return err
+		// process immediately or offload to the proper peer
+		targetShard := r.Sharder.WhichShard(traceID)
+		if targetShard.Equals(r.Sharder.MyShard()) {
+			ok, _ := r.Collector.ProcessSpanImmediately(span)
+			// ProcessSpanImmediately function handles the metrics
+			if !ok {
+				r.Metrics.Increment(r.incomingOrPeer + "_router_dropped")
+				debugLog.Logf("Dropping span from batch, channel full")
+				return err
+			}
+		} else {
+			ev.APIHost = targetShard.GetAddress()
+			r.PeerTransmission.EnqueueEvent(ev)
+			r.Metrics.Increment(r.incomingOrPeer + "_router_peer")
+			debugLog.Logf("redirected span due to AddSpan channel being full")
+			return err
+		}
 	}
 
 	r.Metrics.Increment(r.incomingOrPeer + "_router_span")

From cfc96732be9fc431d734d03bfa62fbfc25004d8a Mon Sep 17 00:00:00 2001
From: Mike Terhar <mike@terhar.com>
Date: Mon, 14 Apr 2025 13:46:18 +0000
Subject: [PATCH 2/2] add _sum to legacy metrics histograms

---
 metrics/legacy.go | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/metrics/legacy.go b/metrics/legacy.go
index e2bec8cbd2..87b319076b 100644
--- a/metrics/legacy.go
+++ b/metrics/legacy.go
@@ -266,7 +266,9 @@ func (h *LegacyMetrics) reportToHoneycomb(ctx context.Context) {
 					ev.AddField(histogram.name+"_p99", histogram.vals[p99Index])
 					ev.AddField(histogram.name+"_min", histogram.vals[0])
 					ev.AddField(histogram.name+"_max", histogram.vals[len(histogram.vals)-1])
-					ev.AddField(histogram.name+"_avg", average(histogram.vals))
+					avg, sum := averageAndSum(histogram.vals)
+					ev.AddField(histogram.name+"_avg", avg)
+					ev.AddField(histogram.name+"_sum", sum)
 					histogram.vals = histogram.vals[:0]
 				}
 				histogram.lock.Unlock()
@@ -306,6 +308,14 @@ func average(vals []float64) float64 {
 	return total / float64(len(vals))
 }
 
+func averageAndSum(vals []float64) (float64, float64) {
+	var total float64
+	for _, val := range vals {
+		total += val
+	}
+	return total / float64(len(vals)), total
+}
+
 func (h *LegacyMetrics) Register(metadata Metadata) {
 	h.Logger.Debug().Logf("metrics registering %s with name %s", metadata.Type, metadata.Name)
 	switch metadata.Type {