Add opt-in retry-with-backoff for rejected pushes

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## Unreleased
+
+This release adds automatic retry for `ImageUpdateAutomation` pushes rejected
+because another writer already advanced the same push branch (a lost
+non-fast-forward race). On a rejected push, the controller now fetches and
+hard-resets to the new remote tip, re-applies policies, and retries the
+commit and push, up to 5 attempts with exponential backoff (2s/4s/8s/16s),
+instead of waiting for the next scheduled reconciliation. This is controlled
+by the new `GitPushRetryOnConflict` feature gate, disabled by default.
+
+Improvements:
+- Retry pushes rejected due to a lost push race instead of waiting for the
+  next reconciliation, behind the opt-in `GitPushRetryOnConflict` feature gate
+
 ## 1.2.5
 
 **Release date:** 2026-08-31

go.mod

@@ -202,3 +202,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v6 v6.4.0 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect
 )
+
+replace github.com/fluxcd/pkg/git => github.com/monotek/fluxcd-pkg/git v0.0.0-20260831175242-c1457b1dca05

go.sum

@@ -136,8 +136,6 @@ github.com/fluxcd/pkg/auth v0.54.0 h1:EiNUhksFwUULmrctsTcfjtwszmzVgpEPuP2LcgfwIb
 github.com/fluxcd/pkg/auth v0.54.0/go.mod h1:bf+0mQNaxgMLvdR3S15qz/3t0GoLTipl1xDQBguNsJI=
 github.com/fluxcd/pkg/cache v0.14.0 h1:wEwJA8NhYj+nH9P6ifcsglDZARWlcbxbmwngGOzfU4c=
 github.com/fluxcd/pkg/cache v0.14.0/go.mod h1:KwzU2gyVQ83YOHJsbBeveJ0HsXmLrH0I668zX19d/+s=
-github.com/fluxcd/pkg/git v0.52.0 h1:dgsliHdaLADUcDO4pI0pc11N4dZ21NfDdhNcgRNuAkM=
-github.com/fluxcd/pkg/git v0.52.0/go.mod h1:mOvFDxoiuz+Mm4Ux1wKeTTckvBgZFvbTK8lNxmVHzKs=
 github.com/fluxcd/pkg/gittestserver v0.29.0 h1:2j03zKVL6iVn6oiUuecG/O/3Q1pULWM9JrF/HSjkpnc=
 github.com/fluxcd/pkg/gittestserver v0.29.0/go.mod h1:O8151jV0ppBZTb9IUXMjxh6hZpkiuLq8JQHDBPOkZFw=
 github.com/fluxcd/pkg/runtime v0.110.0 h1:ziGAuoQ3OVSEqmMXS6doZWi2LcF7exEKPe69dun5RNg=
@@ -302,6 +300,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/monotek/fluxcd-pkg/git v0.0.0-20260831175242-c1457b1dca05 h1:v/aprrWOMkIb0IUrckkrXC8+9nTBXDLLcR49ON32zUA=
+github.com/monotek/fluxcd-pkg/git v0.0.0-20260831175242-c1457b1dca05/go.mod h1:mOvFDxoiuz+Mm4Ux1wKeTTckvBgZFvbTK8lNxmVHzKs=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI=

internal/controller/imageupdateautomation_controller.go

@@ -491,7 +491,7 @@ func (r *ImageUpdateAutomationReconciler) reconcile(ctx context.Context, sp *pat
 		pushCfg = append(pushCfg, source.WithPushConfigOptions(obj.Spec.GitSpec.Push.Options))
 	}
 
-	pushResult, err = sm.CommitAndPush(ctx, obj, policyResult, pushCfg...)
+	pushResult, err = r.commitAndPushWithRetry(ctx, sm, obj, policies, policyResult, pushCfg)
 	if err != nil {
 		// Check if error is due to removed template field usage.
 		// Set Stalled condition and return nil error to prevent requeue, allowing user to fix template.

internal/controller/push_retry.go

@@ -0,0 +1,128 @@
+/*
+Copyright 2026 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	reflectorv1 "github.com/fluxcd/image-reflector-controller/api/v1"
+
+	imagev1 "github.com/fluxcd/image-automation-controller/api/v1"
+	"github.com/fluxcd/image-automation-controller/internal/features"
+	"github.com/fluxcd/image-automation-controller/internal/policy"
+	"github.com/fluxcd/image-automation-controller/internal/source"
+	"github.com/fluxcd/image-automation-controller/internal/update"
+)
+
+// These are vars, not consts, so tests can shrink them for determinism and
+// speed without changing the retry mechanics itself.
+var (
+	// pushRetryMaxAttempts mirrors Staffbase's gitops-github-action
+	// retry_with_backoff(5, 2, push_to_gitops_repo) pattern: 5 attempts,
+	// base delay 2s, doubling each retry.
+	pushRetryMaxAttempts = 5
+	// pushRetryBaseDelay is the initial backoff, doubled after each retry:
+	// 2s, 4s, 8s, 16s.
+	pushRetryBaseDelay = 2 * time.Second
+	// maxPushRetryBudget bounds how long a single reconcile() call may block
+	// retrying a push, independent of the object's configured interval, so
+	// heavy contention on one branch cannot starve other ImageUpdateAutomation
+	// objects sharing the same reconcile worker pool.
+	maxPushRetryBudget = 2 * time.Minute
+)
+
+// pushRetryTestHook, when set, is called immediately before each attempt's
+// CommitAndPush. It exists only so tests can deterministically land a
+// competing commit inside the race window, instead of relying on wall-clock
+// timing against a background pusher. Always nil in production.
+var pushRetryTestHook func(attempt int)
+
+// commitAndPushWithRetry wraps SourceManager.CommitAndPush with a
+// fetch+reset+reapply retry loop, engaged only when the push is rejected
+// because another writer already advanced the same branch
+// (source.IsPushConflict). Any other error, or exhaustion of
+// pushRetryMaxAttempts, is returned unchanged for the caller's existing
+// error handling.
+//
+// On retry, only the cheap SourceManager.RefreshToRemote (fetch + hard
+// reset) is used to catch the working directory up to the new remote tip;
+// SourceManager.CheckoutSource's full clone is deliberately not repeated.
+func (r *ImageUpdateAutomationReconciler) commitAndPushWithRetry(
+	ctx context.Context,
+	sm *source.SourceManager,
+	obj *imagev1.ImageUpdateAutomation,
+	policies []reflectorv1.ImagePolicy,
+	policyResult update.Result,
+	pushCfg []source.PushConfig,
+) (*source.PushResult, error) {
+	if !r.features[features.GitPushRetryOnConflict] {
+		return sm.CommitAndPush(ctx, obj, policyResult, pushCfg...)
+	}
+
+	retryBudget := obj.GetRequeueAfter() / 2
+	if retryBudget > maxPushRetryBudget {
+		retryBudget = maxPushRetryBudget
+	}
+	retryCtx, cancel := context.WithTimeout(ctx, retryBudget)
+	defer cancel()
+
+	log := ctrl.LoggerFrom(ctx)
+
+	for attempt := 1; attempt <= pushRetryMaxAttempts; attempt++ {
+		if pushRetryTestHook != nil {
+			pushRetryTestHook(attempt)
+		}
+		pushResult, err := sm.CommitAndPush(retryCtx, obj, policyResult, pushCfg...)
+		if err == nil {
+			if attempt > 1 {
+				log.Info("push succeeded after retry", "attempts", attempt)
+			}
+			return pushResult, nil
+		}
+		if !source.IsPushConflict(err) || attempt == pushRetryMaxAttempts {
+			return nil, err
+		}
+
+		delay := pushRetryBaseDelay * time.Duration(uint64(1)<<uint(attempt-1)) // 2s, 4s, 8s, 16s
+		log.Info("push rejected, remote branch has moved; retrying after refresh",
+			"attempt", attempt, "maxAttempts", pushRetryMaxAttempts, "delay", delay)
+
+		select {
+		case <-time.After(delay):
+		case <-retryCtx.Done():
+			return nil, fmt.Errorf("push retry budget exceeded after %d attempt(s): %w", attempt, retryCtx.Err())
+		}
+
+		if err := sm.RefreshToRemote(retryCtx); err != nil {
+			return nil, fmt.Errorf("failed to refresh working tree after push conflict: %w", err)
+		}
+
+		policyResult, err = policy.ApplyPolicies(retryCtx, sm.WorkDirectory(), obj, policies)
+		if err != nil {
+			return nil, err
+		}
+		// If policyResult is now empty (a concurrent commit already achieved
+		// the desired end state), the next loop iteration's CommitAndPush
+		// returns (nil, nil) via its existing no-file-changes handling, which
+		// the err == nil branch above already treats as success.
+	}
+	panic("unreachable: loop always returns before falling through")
+}