Add some jitter to pkg/retry

... so that a short failure does not trigger a cascade
of synchronized retries at exactly the same time.

Keep the expected duration of the retry unchanged,
vary it 5 % up or down.

Signed-off-by: Miloslav Trmač <[email protected]>

Author: mtrmac

libimage/manifests/manifests_test.go

@@ -726,17 +726,19 @@ func TestPushManifest(t *testing.T) {
 	_, _, err = list.Push(ctx, bogusDestRef, options)
 	assert.Error(t, err)
 	logString := logBuffer.String()
-	// Must show warning where libimage is going to retry 5 times with 1s delay
-	assert.Contains(t, logString, "Failed, retrying in 1s ... (1/5)", "warning not matched")
+	// Must show warning where libimage is going to retry 5 times with ~1s delay
+	assert.Regexp(t, `Failed, retrying in \(1\.[0-9]*s\|[0-9].*ms) ... (1/5)`, logString, "warning not matched")
+	// The 4th delay is approximately 8 seconds.
+	assert.Regexp(t, `Failed, retrying in [789].[0-9]*s ... (4/5)`, logString, "warning not matched")
 
 	logBuffer = bytes.Buffer{}
 	logrus.SetOutput(&logBuffer)
 	options.RetryDelay = &delay
 	_, _, err = list.Push(ctx, bogusDestRef, options)
 	assert.Error(t, err)
 	logString = logBuffer.String()
-	// Must show warning where libimage is going to retry 5 times with 3s delay
-	assert.Contains(t, logString, "Failed, retrying in 3s ... (1/5)", "warning not matched")
+	// Must show warning where libimage is going to retry 5 times with ~3s delay
+	assert.Regexp(t, `Failed, retrying in [23].[0-9]*s ... (1/5)`, logString, "warning not matched")
 
 	options.AddCompression = []string{"zstd"}
 	options.ImageListSelection = cp.CopyAllImages

pkg/retry/retry.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 	"math"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"net/url"
@@ -46,6 +47,7 @@ func IfNecessary(ctx context.Context, operation func() error, options *Options)
 		if options.Delay != 0 {
 			delay = options.Delay
 		}
+		delay += rand.N(delay/10) - delay/20 // 10 % jitter so that a failure blip doesn’t cause a deterministic stampede
 		logrus.Warnf("Failed, retrying in %s ... (%d/%d). Error: %v", delay, attempt+1, options.MaxRetry, err)
 		select {
 		case <-time.After(delay):