fix(outputs/plugins/datadog): prevent retry on authentication error

Signed-off-by: Florentin Dubois <florentin.dubois@clever.cloud>

Author: FlorentinDUBOIS

internal/errors.go

@@ -1,6 +1,9 @@
 package internal
 
-import "errors"
+import (
+	"errors"
+	"fmt"
+)
 
 var (
 	ErrNotConnected     = errors.New("not connected")
@@ -61,3 +64,23 @@ func (e *PartialWriteError) Error() string {
 func (e *PartialWriteError) Unwrap() error {
 	return e.Err
 }
+
+// HTTPError represents an error from an HTTP API with retry information.
+// Use this for HTTP-based output plugins to indicate whether an error
+// is retryable (e.g., 5xx server errors) or not (e.g., 4xx client errors).
+type HTTPError struct {
+	Err        error
+	StatusCode int
+	Retryable  bool
+}
+
+func (e *HTTPError) Error() string {
+	if e.Err == nil {
+		return fmt.Sprintf("HTTP error: status %d", e.StatusCode)
+	}
+	return e.Err.Error()
+}
+
+func (e *HTTPError) Unwrap() error {
+	return e.Err
+}

plugins/outputs/datadog/datadog.go

@@ -189,7 +189,20 @@ func (d *Datadog) Write(metrics []telegraf.Metric) error {
 	if resp.StatusCode < 200 || resp.StatusCode > 209 {
 		//nolint:errcheck // err can be ignored since it is just for logging
 		body, _ := io.ReadAll(resp.Body)
-		return fmt.Errorf("received bad status code, %d: %s", resp.StatusCode, string(body))
+		// 4xx client errors are not retryable - the request itself is invalid
+		if resp.StatusCode >= 400 && resp.StatusCode < 500 {
+			d.Log.Errorf("Client error %d, metrics will be dropped: %s", resp.StatusCode, string(body))
+			return &internal.HTTPError{
+				Err:        fmt.Errorf("client error %d: metrics dropped", resp.StatusCode),
+				StatusCode: resp.StatusCode,
+				Retryable:  false,
+			}
+		}
+		return &internal.HTTPError{
+			Err:        fmt.Errorf("received bad status code, %d: %s", resp.StatusCode, string(body)),
+			StatusCode: resp.StatusCode,
+			Retryable:  true,
+		}
 	}
 
 	return nil