tune deployment operator service reconciliation

Author: michaeljguarino

.cursor/skills/console-tests/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: console-tests
-description: Run and interpret tests in the Console Elixir repo. Use when validating changes with `mix test`, running Docker integration tests, diagnosing sandbox network failures, or explaining Console test setup in Cursor.
+description: Run and interpret tests in the Console repo. Use when validating changes with `mix test`, running deployment-operator Go tests, diagnosing sandbox/toolchain/network failures, or explaining Console test setup in Cursor.
 ---
 
 # Console Tests
@@ -10,8 +10,19 @@ Use this skill when validating changes in the Console repo.
 ## Sandbox Requirements
 
 - Run `mix test` commands from the repo root: `/Users/michaelguarino/code/console`.
+- Run deployment-operator Go commands from `/Users/michaelguarino/code/console/go/deployment-operator`.
 - The default Cursor sandbox blocks some domains the Console app touches during startup and test aliases.
 - Use `required_permissions: ["full_network"]` for `mix test` runs that need the normal project setup.
+- For Go tests, explicitly set Go caches inside the workspace when running in the sandbox:
+
+```bash
+cd /Users/michaelguarino/code/console/go/deployment-operator && \
+GOPATH=/Users/michaelguarino/code/console/.go \
+GOCACHE=/Users/michaelguarino/code/console/.cache/go-build \
+go test ./pkg/controller/service
+```
+
+- Use `required_permissions: ["full_network"]` for Go tests when dependencies or the configured Go toolchain need to download. If Go still fails writing to `/Users/michaelguarino/go/pkg/...`, rerun with the in-workspace `GOPATH`/`GOCACHE` above before requesting `all`.
 - Do not run `mix format` in this repository.
 
 ## Why Full Network Is Needed
@@ -38,16 +49,36 @@ Full suite:
 mix test
 ```
 
+Focused deployment-operator service package:
+
+```bash
+cd /Users/michaelguarino/code/console/go/deployment-operator && \
+GOPATH=/Users/michaelguarino/code/console/.go \
+GOCACHE=/Users/michaelguarino/code/console/.cache/go-build \
+go test ./pkg/controller/service
+```
+
+Format changed deployment-operator Go files:
+
+```bash
+cd /Users/michaelguarino/code/console/go/deployment-operator && \
+gofmt -w pkg/controller/service/reconciler.go pkg/manifests/tarball.go
+```
+
 
 ## Interpreting Failures
 
 - A focused test passing is good validation for narrow changes, especially when the full suite is known to depend on external/local fixtures.
 - Full-suite failures like `could not resolve ref main` or `could not resolve ref master` usually indicate git fixture/ref setup problems unless the touched code is in that path.
+- Go failures like `go.work requires go >= 1.26.5` can mean the IDE/linter is using an older local Go. A shell `go test` may still work through Go's auto toolchain download when network and cache permissions are set correctly.
+- Sandbox errors writing under `/Users/michaelguarino/go/pkg/sumdb` indicate the Go module/toolchain cache is outside the writable workspace. Prefer setting `GOPATH` and `GOCACHE` inside `/Users/michaelguarino/code/console`.
 - Report full-suite failures honestly, but separate environment/fixture failures from failures in the files being changed.
 
 ## Validation Checklist
 
 - [ ] Run the most relevant focused test file with `full_network` if it uses project startup or external services.
+- [ ] For deployment-operator Go changes, run `gofmt -w` on touched files from the module directory.
+- [ ] For deployment-operator Go tests, `cd` into `go/deployment-operator` and use in-workspace `GOPATH`/`GOCACHE` when sandboxed.
 - [ ] Run the full suite with `full_network` when requested or when the change has broad impact.
 - [ ] Read the final ExUnit summary and any failure blocks for touched modules.
 - [ ] Run `ReadLints` on changed files after edits.

go/deployment-operator/pkg/controller/service/reconciler.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"strings"
 	"time"
@@ -387,7 +388,7 @@ func (s *ServiceReconciler) Poll(ctx context.Context) error {
 			logger.V(4).Info("enqueueing update for", "service", svc.Node.ID)
 			s.svcCache.Add(svc.Node.ID, svc.Node)
 			currentServices.Add(svc.Node.Name)
-			s.svcQueue.AddAfter(svc.Node.ID, utils.Jitter(15*time.Second))
+			s.svcQueue.AddAfter(svc.Node.ID, utils.Jitter(s.pollJitterWindow()))
 		}
 	}
 
@@ -489,6 +490,10 @@ func (s *ServiceReconciler) Reconcile(ctx context.Context, id string) (result re
 	dir, err := s.manifestCache.Fetch(svc)
 	if err != nil {
 		logger.Error(err, "failed to parse manifests", "service", svc.Name)
+		if isTransientFetchError(err) {
+			done = true
+			return ctrl.Result{}, nil
+		}
 		if isExpectedError(err) {
 			// mark as the expected error so that it won't get propagated to the API as a service error
 			err = plrlerrors.ErrExpected
@@ -615,3 +620,25 @@ func isExpectedError(err error) bool {
 	}
 	return false
 }
+
+func isRateLimitError(err error) bool {
+	var httpErr *manis.HTTPError
+	return errors.As(err, &httpErr) && httpErr.StatusCode == http.StatusTooManyRequests
+}
+
+func isTimeoutError(err error) bool {
+	var netErr net.Error
+	return errors.As(err, &netErr) && netErr.Timeout()
+}
+
+func isTransientFetchError(err error) bool {
+	return isRateLimitError(err) || isTimeoutError(err)
+}
+
+func (s *ServiceReconciler) pollJitterWindow() time.Duration {
+	interval := s.GetPollInterval()()
+	if interval <= 0 {
+		return 15 * time.Second
+	}
+	return interval / 2
+}

go/deployment-operator/pkg/manifests/tarball.go

@@ -14,7 +14,7 @@ import (
 const pluralDigestHeader = "x-plrl-digest"
 
 var (
-	timeout = 60 * time.Second
+	timeout = 10 * time.Second
 	client  = &http.Client{
 		Timeout: timeout,
 		Transport: &http.Transport{