[exporter/otlphttp] Add non_retryable_status to skip retries for specific HTTP codes

Fixes #14228

Author: farhan-pasha

.chloggen/otlphttp-non-retryable-status.yaml

@@ -0,0 +1,30 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'enhancement'
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/otlp)
+component: exporter/otlphttp
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add `non_retryable_status` configuration option to prevent internal retries for specific HTTP status codes.
+
+# One or more tracking issues or pull requests related to the change
+issues: [14228]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  The new `retry_on_failure.non_retryable_status` field allows configuring HTTP status codes
+  (e.g., 429, 502, 503, 504) that should NOT trigger internal retries. When configured, these
+  status codes are treated as permanent errors and data is dropped immediately instead of being
+  queued and retried. This is useful in gateway mode deployments to prevent queue buildup when
+  backend services return rate limit or temporary failure responses.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/otlphttpexporter/README.md

@@ -42,6 +42,7 @@ The following settings can be optionally configured:
 - `write_buffer_size` (default = 512 * 1024): WriteBufferSize for HTTP client.
 - `encoding` (default = proto): The encoding to use for the messages (valid options: `proto`, `json`)
 - `retry_on_failure`:  see [Retry on Failure](../exporterhelper/README.md#retry-on-failure) for the full set of available options.
+  - `non_retryable_status` (default = []): List of HTTP status codes that should NOT trigger retries. By default, the exporter retries on 429, 502, 503, and 504. This option allows marking these codes as permanent errors to prevent internal retries. Useful in gateway mode to prevent queue buildup when backends return rate limit errors.
 - `sending_queue`: see [Sending Queue](../exporterhelper/README.md#sending-queue) for the full set of available options.
 
 Example:
@@ -70,5 +71,16 @@ exporters:
     encoding: json
 ```
 
+To prevent internal retries for specific HTTP status codes (useful in gateway mode):
+
+```yaml
+exporters:
+  otlphttp:
+    endpoint: https://backend:4318
+    retry_on_failure:
+      enabled: true
+      non_retryable_status: [429, 503]  # Don't retry rate limits and service unavailable
+```
+
 The full list of settings exposed for this exporter are documented [here](./config.go)
 with detailed sample configurations [here](./testdata/config.yaml).

exporter/otlphttpexporter/config.go

@@ -44,11 +44,21 @@ func (e *EncodingType) UnmarshalText(text []byte) error {
 	return nil
 }
 
+// RetryConfig extends the standard BackOffConfig with additional HTTP-specific options.
+type RetryConfig struct {
+	configretry.BackOffConfig `mapstructure:",squash"`
+
+	// NonRetryableStatus is a list of HTTP status codes that should NOT be retried.
+	// By default, the exporter retries on 429, 502, 503, and 504 per OTLP spec.
+	// Use this to treat specific retryable codes as permanent errors.
+	NonRetryableStatus []int `mapstructure:"non_retryable_status"`
+}
+
 // Config defines configuration for OTLP/HTTP exporter.
 type Config struct {
 	ClientConfig confighttp.ClientConfig                                  `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
 	QueueConfig  configoptional.Optional[exporterhelper.QueueBatchConfig] `mapstructure:"sending_queue"`
-	RetryConfig  configretry.BackOffConfig                                `mapstructure:"retry_on_failure"`
+	RetryConfig  RetryConfig                                              `mapstructure:"retry_on_failure"`
 
 	// The URL to send traces to. If omitted the Endpoint + "/v1/traces" will be used.
 	TracesEndpoint string `mapstructure:"traces_endpoint"`
@@ -73,5 +83,13 @@ func (cfg *Config) Validate() error {
 	if cfg.ClientConfig.Endpoint == "" && cfg.TracesEndpoint == "" && cfg.MetricsEndpoint == "" && cfg.LogsEndpoint == "" && cfg.ProfilesEndpoint == "" {
 		return errors.New("at least one endpoint must be specified")
 	}
+
+	// Validate non-retryable status codes
+	for _, code := range cfg.RetryConfig.NonRetryableStatus {
+		if code < 100 || code > 599 {
+			return fmt.Errorf("invalid HTTP status code in retry_on_failure.non_retryable_status: %d (must be between 100-599)", code)
+		}
+	}
+
 	return nil
 }

exporter/otlphttpexporter/config_test.go

@@ -45,13 +45,15 @@ func TestUnmarshalConfig(t *testing.T) {
 	require.NoError(t, cm.Unmarshal(&cfg))
 	assert.Equal(t,
 		&Config{
-			RetryConfig: configretry.BackOffConfig{
-				Enabled:             true,
-				InitialInterval:     10 * time.Second,
-				RandomizationFactor: 0.7,
-				Multiplier:          1.3,
-				MaxInterval:         1 * time.Minute,
-				MaxElapsedTime:      10 * time.Minute,
+			RetryConfig: RetryConfig{
+				BackOffConfig: configretry.BackOffConfig{
+					Enabled:             true,
+					InitialInterval:     10 * time.Second,
+					RandomizationFactor: 0.7,
+					Multiplier:          1.3,
+					MaxInterval:         1 * time.Minute,
+					MaxElapsedTime:      10 * time.Minute,
+				},
 			},
 			QueueConfig: configoptional.Some(exporterhelper.QueueBatchConfig{
 				Sizer:        exporterhelper.RequestSizerTypeRequests,
@@ -196,14 +198,73 @@ func TestConfigValidate(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "valid non-retryable status codes",
+			cfg: &Config{
+				ClientConfig: confighttp.ClientConfig{
+					Endpoint: "http://localhost:4318",
+				},
+				RetryConfig: RetryConfig{
+					NonRetryableStatus: []int{429, 503},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "empty non-retryable status list",
+			cfg: &Config{
+				ClientConfig: confighttp.ClientConfig{
+					Endpoint: "http://localhost:4318",
+				},
+				RetryConfig: RetryConfig{
+					NonRetryableStatus: []int{},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "invalid status code too low",
+			cfg: &Config{
+				ClientConfig: confighttp.ClientConfig{
+					Endpoint: "http://localhost:4318",
+				},
+				RetryConfig: RetryConfig{
+					NonRetryableStatus: []int{99},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "invalid status code too high",
+			cfg: &Config{
+				ClientConfig: confighttp.ClientConfig{
+					Endpoint: "http://localhost:4318",
+				},
+				RetryConfig: RetryConfig{
+					NonRetryableStatus: []int{600},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "invalid status code in list",
+			cfg: &Config{
+				ClientConfig: confighttp.ClientConfig{
+					Endpoint: "http://localhost:4318",
+				},
+				RetryConfig: RetryConfig{
+					NonRetryableStatus: []int{429, 0, 503},
+				},
+			},
+			wantErr: true,
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			err := tt.cfg.Validate()
 			if tt.wantErr {
 				require.Error(t, err)
-				assert.Contains(t, err.Error(), "at least one endpoint must be specified")
 			} else {
 				assert.NoError(t, err)
 			}

exporter/otlphttpexporter/factory.go

@@ -44,7 +44,10 @@ func createDefaultConfig() component.Config {
 	clientConfig.WriteBufferSize = 512 * 1024
 
 	return &Config{
-		RetryConfig:  configretry.NewDefaultBackOffConfig(),
+		RetryConfig: RetryConfig{
+			BackOffConfig:      configretry.NewDefaultBackOffConfig(),
+			NonRetryableStatus: nil, // Empty by default - preserves existing retry behavior for 429, 502, 503, 504
+		},
 		QueueConfig:  configoptional.Some(exporterhelper.NewDefaultQueueConfig()),
 		Encoding:     EncodingProto,
 		ClientConfig: clientConfig,
@@ -96,7 +99,7 @@ func createTraces(
 		exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: false}),
 		// explicitly disable since we rely on http.Client timeout logic.
 		exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0}),
-		exporterhelper.WithRetry(oCfg.RetryConfig),
+		exporterhelper.WithRetry(oCfg.RetryConfig.BackOffConfig),
 		exporterhelper.WithQueue(oCfg.QueueConfig))
 }
 
@@ -122,7 +125,7 @@ func createMetrics(
 		exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: false}),
 		// explicitly disable since we rely on http.Client timeout logic.
 		exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0}),
-		exporterhelper.WithRetry(oCfg.RetryConfig),
+		exporterhelper.WithRetry(oCfg.RetryConfig.BackOffConfig),
 		exporterhelper.WithQueue(oCfg.QueueConfig))
 }
 
@@ -147,7 +150,7 @@ func createLogs(
 		exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: false}),
 		// explicitly disable since we rely on http.Client timeout logic.
 		exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0}),
-		exporterhelper.WithRetry(oCfg.RetryConfig),
+		exporterhelper.WithRetry(oCfg.RetryConfig.BackOffConfig),
 		exporterhelper.WithQueue(oCfg.QueueConfig))
 }
 
@@ -173,6 +176,6 @@ func createProfiles(
 		exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: false}),
 		// explicitly disable since we rely on http.Client timeout logic.
 		exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0}),
-		exporterhelper.WithRetry(oCfg.RetryConfig),
+		exporterhelper.WithRetry(oCfg.RetryConfig.BackOffConfig),
 		exporterhelper.WithQueue(oCfg.QueueConfig))
 }

exporter/otlphttpexporter/otlp.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"net/url"
 	"runtime"
+	"slices"
 	"strconv"
 	"time"
 
@@ -222,7 +223,7 @@ func (e *baseExporter) export(ctx context.Context, url string, request []byte, p
 	}
 	formattedErr = statusutil.NewStatusFromMsgAndHTTPCode(errString, resp.StatusCode).Err()
 
-	if !isRetryableStatusCode(resp.StatusCode) {
+	if !e.isRetryableStatusCode(resp.StatusCode) {
 		return consumererror.NewPermanent(formattedErr)
 	}
 
@@ -253,7 +254,13 @@ func (e *baseExporter) export(ctx context.Context, url string, request []byte, p
 
 // Determine if the status code is retryable according to the specification.
 // For more, see https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#failures-1
-func isRetryableStatusCode(code int) bool {
+func (e *baseExporter) isRetryableStatusCode(code int) bool {
+	// First, check if this status code is explicitly configured as non-retryable
+	if slices.Contains(e.config.RetryConfig.NonRetryableStatus, code) {
+		return false
+	}
+
+	// Then check the default retryable status codes per OTLP spec
 	switch code {
 	case http.StatusTooManyRequests:
 		return true