Add detailed failure attributes to exporter send_failed metrics

Signed-off-by: Israel Blancas <iblancasa@gmail.com>

fix

Signed-off-by: Israel Blancas <iblancasa@gmail.com>

Author: iblancasa

.chloggen/13956.yaml

@@ -0,0 +1,30 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/otlp)
+component: all
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add detailed failure attributes to exporter send_failed metrics at detailed telemetry level.
+
+# One or more tracking issues or pull requests related to the change
+issues: [13956]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |-
+  The `otelcol_exporter_send_failed_{spans,metric_points,log_records}` metrics now include detailed
+  failure attributes when telemetry level is Detailed or higher: `failure.reason` (categorizes
+  failure reason) and `failure.permanent` (indicates if error is permanent).
+  This enables better alerting and debugging by distinguishing between retries exhausted, permanent
+  errors, shutdown, context cancellation, and transient errors.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/exporterhelper/documentation.md

@@ -64,23 +64,23 @@ Current size of the retry queue (in batches). [Alpha]
 
 ### otelcol_exporter_send_failed_log_records
 
-Number of log records in failed attempts to send to destination. [Alpha]
+Number of log records in failed attempts to send to destination. At detailed telemetry level, includes attributes: failure.reason, failure.permanent. [Alpha]
 
 | Unit | Metric Type | Value Type | Monotonic | Stability |
 | ---- | ----------- | ---------- | --------- | --------- |
 | {records} | Sum | Int | true | Alpha |
 
 ### otelcol_exporter_send_failed_metric_points
 
-Number of metric points in failed attempts to send to destination. [Alpha]
+Number of metric points in failed attempts to send to destination. At detailed telemetry level, includes attributes: failure.reason, failure.permanent. [Alpha]
 
 | Unit | Metric Type | Value Type | Monotonic | Stability |
 | ---- | ----------- | ---------- | --------- | --------- |
 | {datapoints} | Sum | Int | true | Alpha |
 
 ### otelcol_exporter_send_failed_spans
 
-Number of spans in failed attempts to send to destination. [Alpha]
+Number of spans in failed attempts to send to destination. At detailed telemetry level, includes attributes: failure.reason, failure.permanent. [Alpha]
 
 | Unit | Metric Type | Value Type | Monotonic | Stability |
 | ---- | ----------- | ---------- | --------- | --------- |

exporter/exporterhelper/go.mod

@@ -10,6 +10,7 @@ require (
 	go.opentelemetry.io/collector/component/componenttest v0.141.0
 	go.opentelemetry.io/collector/config/configoptional v1.47.0
 	go.opentelemetry.io/collector/config/configretry v1.47.0
+	go.opentelemetry.io/collector/config/configtelemetry v0.141.0
 	go.opentelemetry.io/collector/confmap v1.47.0
 	go.opentelemetry.io/collector/confmap/xconfmap v0.141.0
 	go.opentelemetry.io/collector/consumer v1.47.0
@@ -89,6 +90,8 @@ replace go.opentelemetry.io/collector/receiver => ../../receiver
 
 replace go.opentelemetry.io/collector/config/configretry => ../../config/configretry
 
+replace go.opentelemetry.io/collector/config/configtelemetry => ../../config/configtelemetry
+
 replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer
 
 replace go.opentelemetry.io/collector/consumer/consumertest => ../../consumer/consumertest

exporter/exporterhelper/internal/metadata/generated_telemetry.go

@@ -5,14 +5,19 @@ package internal // import "go.opentelemetry.io/collector/exporter/exporterhelpe
 
 import (
 	"context"
+	"errors"
+	"strings"
 
 	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/codes"
+	otelcodes "go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/trace"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/configtelemetry"
+	"go.opentelemetry.io/collector/consumer/consumererror"
 	"go.opentelemetry.io/collector/exporter"
+	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/experr"
 	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/metadata"
 	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/queuebatch"
 	"go.opentelemetry.io/collector/exporter/exporterhelper/internal/request"
@@ -36,6 +41,15 @@ const (
 	ItemsFailed = "items.failed"
 )
 
+// FailureAttributeKey represents attribute keys for detailed failure information.
+// These attributes are only emitted at detailed telemetry level.
+type FailureAttributeKey string
+
+const (
+	FailureReasonKey    FailureAttributeKey = "failure.reason"
+	FailurePermanentKey FailureAttributeKey = "failure.permanent"
+)
+
 type obsReportSender[K request.Request] struct {
 	component.StartFunc
 	component.ShutdownFunc
@@ -46,6 +60,8 @@ type obsReportSender[K request.Request] struct {
 	metricAttr      metric.MeasurementOption
 	itemsSentInst   metric.Int64Counter
 	itemsFailedInst metric.Int64Counter
+	telemetryLevel  configtelemetry.Level
+	exporterID      string
 	next            sender.Sender[K]
 }
 
@@ -57,13 +73,16 @@ func newObsReportSender[K request.Request](set exporter.Settings, signal pipelin
 
 	idStr := set.ID.String()
 	expAttr := attribute.String(ExporterKey, idStr)
+	telemetryLevel := configtelemetry.LevelDetailed
 
 	or := &obsReportSender[K]{
-		spanName:   ExporterKey + spanNameSep + idStr + spanNameSep + signal.String(),
-		tracer:     metadata.Tracer(set.TelemetrySettings),
-		spanAttrs:  trace.WithAttributes(expAttr, attribute.String(DataTypeKey, signal.String())),
-		metricAttr: metric.WithAttributeSet(attribute.NewSet(expAttr)),
-		next:       next,
+		spanName:       ExporterKey + spanNameSep + idStr + spanNameSep + signal.String(),
+		tracer:         metadata.Tracer(set.TelemetrySettings),
+		spanAttrs:      trace.WithAttributes(expAttr, attribute.String(DataTypeKey, signal.String())),
+		metricAttr:     metric.WithAttributeSet(attribute.NewSet(expAttr)),
+		telemetryLevel: telemetryLevel,
+		exporterID:     idStr,
+		next:           next,
 	}
 
 	switch signal {
@@ -112,9 +131,15 @@ func (ors *obsReportSender[K]) endOp(ctx context.Context, numLogRecords int, err
 	if ors.itemsSentInst != nil {
 		ors.itemsSentInst.Add(ctx, numSent, ors.metricAttr)
 	}
-	// No metrics recorded for profiles.
-	if ors.itemsFailedInst != nil {
-		ors.itemsFailedInst.Add(ctx, numFailedToSend, ors.metricAttr)
+	if ors.itemsFailedInst != nil && numFailedToSend > 0 {
+		if ors.telemetryLevel >= configtelemetry.LevelDetailed {
+			failedAttrs := extractFailureAttributes(err)
+			baseAttrs := attribute.NewSet(attribute.String(ExporterKey, ors.exporterID))
+			combinedAttrs := attribute.NewSet(append(baseAttrs.ToSlice(), failedAttrs.ToSlice()...)...)
+			ors.itemsFailedInst.Add(ctx, numFailedToSend, metric.WithAttributeSet(combinedAttrs))
+		} else {
+			ors.itemsFailedInst.Add(ctx, numFailedToSend, ors.metricAttr)
+		}
 	}
 
 	span := trace.SpanFromContext(ctx)
@@ -126,7 +151,7 @@ func (ors *obsReportSender[K]) endOp(ctx context.Context, numLogRecords int, err
 			attribute.Int64(ItemsFailed, numFailedToSend),
 		)
 		if err != nil {
-			span.SetStatus(codes.Error, err.Error())
+			span.SetStatus(otelcodes.Error, err.Error())
 		}
 	}
 }
@@ -137,3 +162,38 @@ func toNumItems(numExportedItems int, err error) (int64, int64) {
 	}
 	return int64(numExportedItems), 0
 }
+
+func extractFailureAttributes(err error) attribute.Set {
+	if err == nil {
+		return attribute.NewSet()
+	}
+
+	attrs := []attribute.KeyValue{}
+	reason := determineFailureReason(err)
+	attrs = append(attrs, attribute.String(string(FailureReasonKey), reason))
+	isPermanent := consumererror.IsPermanent(err)
+	attrs = append(attrs, attribute.Bool(string(FailurePermanentKey), isPermanent))
+	return attribute.NewSet(attrs...)
+}
+
+func determineFailureReason(err error) string {
+	if err == nil {
+		return ""
+	}
+	if strings.Contains(err.Error(), "no more retries left") {
+		return "retries_exhausted"
+	}
+	if experr.IsShutdownErr(err) {
+		return "shutdown"
+	}
+	if consumererror.IsPermanent(err) {
+		return "permanent_error"
+	}
+	if errors.Is(err, context.Canceled) {
+		return "context_cancelled"
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		return "context_timeout"
+	}
+	return "transient_error"
+}