tune deployment operator service reconciliation

Author: michaeljguarino

go/deployment-operator/pkg/controller/service/reconciler.go

@@ -387,7 +387,7 @@ func (s *ServiceReconciler) Poll(ctx context.Context) error {
 			logger.V(4).Info("enqueueing update for", "service", svc.Node.ID)
 			s.svcCache.Add(svc.Node.ID, svc.Node)
 			currentServices.Add(svc.Node.Name)
-			s.svcQueue.AddAfter(svc.Node.ID, utils.Jitter(15*time.Second))
+			s.svcQueue.AddAfter(svc.Node.ID, utils.Jitter(s.pollJitterWindow()))
 		}
 	}
 
@@ -489,6 +489,11 @@ func (s *ServiceReconciler) Reconcile(ctx context.Context, id string) (result re
 	dir, err := s.manifestCache.Fetch(svc)
 	if err != nil {
 		logger.Error(err, "failed to parse manifests", "service", svc.Name)
+		if isRateLimitError(err) {
+			// The manifest fetcher already retries 429s; do not add a second workqueue backoff loop.
+			done = true
+			return ctrl.Result{}, nil
+		}
 		if isExpectedError(err) {
 			// mark as the expected error so that it won't get propagated to the API as a service error
 			err = plrlerrors.ErrExpected
@@ -615,3 +620,16 @@ func isExpectedError(err error) bool {
 	}
 	return false
 }
+
+func isRateLimitError(err error) bool {
+	var httpErr *manis.HTTPError
+	return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusTooManyRequests
+}
+
+func (s *ServiceReconciler) pollJitterWindow() time.Duration {
+	interval := s.GetPollInterval()()
+	if interval <= 0 {
+		return 15 * time.Second
+	}
+	return interval / 2
+}