fix: cap startup memory to avoid OOMKill on rollout (#161)

compute-manager was OOMKilled during startup after a rollout restart on
prod: concurrent informer/cache sync across all project control planes
briefly overshot the 1500Mi cgroup limit before the Go GC caught up. The
Go runtime had no awareness of the cgroup limit and steady-state working
set stays well under it, so this was a startup allocation spike, not a leak.

Two mitigations:

- Set GOMEMLIMIT from the cgroup (via automemlimit, ratio 0.9) so the GC
  backpressures before the kernel OOMKills the process. No-op when
  GOMEMLIMIT is already set in the environment (an explicit manifest
  override wins) or when off-cgroup (local dev).
- Strip managedFields from every controller cache (DefaultTransform =
  cache.TransformStripManagedFields()) across the deployment cluster, the
  main multicluster manager, the discovery manager (unstructured, where
  managedFields are heaviest), the per-project clusters that fan in at
  startup, and the federation manager. managedFields are server-side apply
  bookkeeping the controllers never read.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: ecv

cmd/main.go

@@ -11,6 +11,7 @@ import (
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
+	"github.com/KimMachineGun/automemlimit/memlimit"
 	"golang.org/x/sync/errgroup"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
@@ -21,6 +22,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -136,6 +138,21 @@ func main() {
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
+	// Set GOMEMLIMIT from the container's cgroup memory limit so the Go GC
+	// backpressures before the kernel OOMKills the process. Without this the
+	// runtime is blind to the cgroup limit and lets the heap overshoot during
+	// allocation-heavy startup (concurrent informer/cache sync across project
+	// control planes), which is what OOMKilled compute-manager on rollout (#161).
+	//
+	// This is a no-op when GOMEMLIMIT is already set in the environment or
+	// AUTOMEMLIMIT=off, so an explicit manifest override always wins. Off-cgroup
+	// (e.g. local dev on macOS) the provider errors; we log and continue.
+	if limit, err := memlimit.SetGoMemLimitWithOpts(memlimit.WithRatio(0.9)); err != nil {
+		setupLog.Info("GOMEMLIMIT not set from cgroup; using Go default", "reason", err.Error())
+	} else if limit > 0 {
+		setupLog.Info("GOMEMLIMIT set from cgroup", "bytes", limit)
+	}
+
 	if federationKubeconfig != "" {
 		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
 			&clientcmd.ClientConfigLoadingRules{ExplicitPath: federationKubeconfig},
@@ -193,6 +210,11 @@ func main() {
 
 	deploymentCluster, err := cluster.New(cfg, func(o *cluster.Options) {
 		o.Scheme = scheme
+		// Strip managedFields from every cached object. They are pure server-side
+		// apply bookkeeping the controllers never read, and on large/unstructured
+		// objects they dominate the per-object footprint. Dropping them shrinks the
+		// startup cache-sync spike that OOMKilled compute-manager (#161).
+		o.Cache.DefaultTransform = cache.TransformStripManagedFields()
 	})
 	if err != nil {
 		setupLog.Error(err, "failed creating local cluster")
@@ -230,6 +252,7 @@ func main() {
 
 	mgr, err := mcmanager.New(cfg, provider, ctrl.Options{
 		Scheme:                  scheme,
+		Cache:                   cache.Options{DefaultTransform: cache.TransformStripManagedFields()},
 		Metrics:                 metricsServerOptions,
 		WebhookServer:           webhookServer,
 		HealthProbeBindAddress:  probeAddr,
@@ -405,6 +428,12 @@ func initializeClusterDiscovery(
 
 		discoveryManager, err := manager.New(discoveryRestConfig, manager.Options{
 			Metrics: metricsserver.Options{BindAddress: "0"},
+			Cache: cache.Options{
+				// Unstructured objects carry the full managedFields tree in their
+				// content map, so stripping it here is especially impactful for the
+				// discovery cache (#161).
+				DefaultTransform: cache.TransformStripManagedFields(),
+			},
 			Client: client.Options{
 				Cache: &client.CacheOptions{
 					Unstructured: true,
@@ -419,6 +448,11 @@ func initializeClusterDiscovery(
 			ClusterOptions: []cluster.Option{
 				func(o *cluster.Options) {
 					o.Scheme = scheme
+					// Applied to every per-project cluster cache. These are the
+					// caches that fan in concurrently at startup, so stripping
+					// managedFields here is the largest single lever on the
+					// startup memory spike (#161).
+					o.Cache.DefaultTransform = cache.TransformStripManagedFields()
 				},
 			},
 			InternalServiceDiscovery: serverConfig.Discovery.InternalServiceDiscovery,
@@ -486,6 +520,7 @@ func setupManagementControllers(mgr mcmanager.Manager, federationClient client.C
 	// directly anywhere a watchable federation cluster source is required.
 	federationMgr, err := manager.New(federationRestConfig, manager.Options{
 		Scheme:  scheme,
+		Cache:   cache.Options{DefaultTransform: cache.TransformStripManagedFields()},
 		Metrics: metricsserver.Options{BindAddress: "0"},
 	})
 	if err != nil {

go.mod

@@ -3,7 +3,7 @@ module go.datum.net/compute
 go 1.25.0
 
 require (
-	github.com/go-logr/logr v1.4.3
+	github.com/KimMachineGun/automemlimit v0.7.5
 	github.com/google/go-cmp v0.7.0
 	github.com/karmada-io/api v1.15.0
 	github.com/onsi/ginkgo/v2 v2.27.2
@@ -39,6 +39,7 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.1 // indirect
@@ -58,6 +59,7 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect

go.sum

@@ -1,5 +1,7 @@
 cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
 cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
+github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43DwZrJqVwyveTk=
+github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM=
 github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
 github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
 github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
@@ -107,6 +109,8 @@ github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns
 github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
 github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
 github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=