Skip to content

Commit c0e1792

Browse files
authored
Merge pull request #162 from datum-cloud/fix/compute-manager-startup-oom-161
2 parents 7f880e5 + 38b8ea9 commit c0e1792

3 files changed

Lines changed: 42 additions & 1 deletion

File tree

cmd/main.go

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
1313
// to ensure that exec-entrypoint and run can make use of them.
14+
"github.com/KimMachineGun/automemlimit/memlimit"
1415
"golang.org/x/sync/errgroup"
1516
_ "k8s.io/client-go/plugin/pkg/client/auth"
1617

@@ -21,6 +22,7 @@ import (
2122
"k8s.io/client-go/rest"
2223
"k8s.io/client-go/tools/clientcmd"
2324
ctrl "sigs.k8s.io/controller-runtime"
25+
"sigs.k8s.io/controller-runtime/pkg/cache"
2426
"sigs.k8s.io/controller-runtime/pkg/client"
2527
"sigs.k8s.io/controller-runtime/pkg/cluster"
2628
"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -136,6 +138,21 @@ func main() {
136138

137139
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
138140

141+
// Set GOMEMLIMIT from the container's cgroup memory limit so the Go GC
142+
// backpressures before the kernel OOMKills the process. Without this the
143+
// runtime is blind to the cgroup limit and lets the heap overshoot during
144+
// allocation-heavy startup (concurrent informer/cache sync across project
145+
// control planes), which is what OOMKilled compute-manager on rollout (#161).
146+
//
147+
// This is a no-op when GOMEMLIMIT is already set in the environment or
148+
// AUTOMEMLIMIT=off, so an explicit manifest override always wins. Off-cgroup
149+
// (e.g. local dev on macOS) the provider errors; we log and continue.
150+
if limit, err := memlimit.SetGoMemLimitWithOpts(memlimit.WithRatio(0.9)); err != nil {
151+
setupLog.Info("GOMEMLIMIT not set from cgroup; using Go default", "reason", err.Error())
152+
} else if limit > 0 {
153+
setupLog.Info("GOMEMLIMIT set from cgroup", "bytes", limit)
154+
}
155+
139156
if federationKubeconfig != "" {
140157
loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
141158
&clientcmd.ClientConfigLoadingRules{ExplicitPath: federationKubeconfig},
@@ -193,6 +210,11 @@ func main() {
193210

194211
deploymentCluster, err := cluster.New(cfg, func(o *cluster.Options) {
195212
o.Scheme = scheme
213+
// Strip managedFields from every cached object. They are pure server-side
214+
// apply bookkeeping the controllers never read, and on large/unstructured
215+
// objects they dominate the per-object footprint. Dropping them shrinks the
216+
// startup cache-sync spike that OOMKilled compute-manager (#161).
217+
o.Cache.DefaultTransform = cache.TransformStripManagedFields()
196218
})
197219
if err != nil {
198220
setupLog.Error(err, "failed creating local cluster")
@@ -230,6 +252,7 @@ func main() {
230252

231253
mgr, err := mcmanager.New(cfg, provider, ctrl.Options{
232254
Scheme: scheme,
255+
Cache: cache.Options{DefaultTransform: cache.TransformStripManagedFields()},
233256
Metrics: metricsServerOptions,
234257
WebhookServer: webhookServer,
235258
HealthProbeBindAddress: probeAddr,
@@ -405,6 +428,12 @@ func initializeClusterDiscovery(
405428

406429
discoveryManager, err := manager.New(discoveryRestConfig, manager.Options{
407430
Metrics: metricsserver.Options{BindAddress: "0"},
431+
Cache: cache.Options{
432+
// Unstructured objects carry the full managedFields tree in their
433+
// content map, so stripping it here is especially impactful for the
434+
// discovery cache (#161).
435+
DefaultTransform: cache.TransformStripManagedFields(),
436+
},
408437
Client: client.Options{
409438
Cache: &client.CacheOptions{
410439
Unstructured: true,
@@ -419,6 +448,11 @@ func initializeClusterDiscovery(
419448
ClusterOptions: []cluster.Option{
420449
func(o *cluster.Options) {
421450
o.Scheme = scheme
451+
// Applied to every per-project cluster cache. These are the
452+
// caches that fan in concurrently at startup, so stripping
453+
// managedFields here is the largest single lever on the
454+
// startup memory spike (#161).
455+
o.Cache.DefaultTransform = cache.TransformStripManagedFields()
422456
},
423457
},
424458
InternalServiceDiscovery: serverConfig.Discovery.InternalServiceDiscovery,
@@ -486,6 +520,7 @@ func setupManagementControllers(mgr mcmanager.Manager, federationClient client.C
486520
// directly anywhere a watchable federation cluster source is required.
487521
federationMgr, err := manager.New(federationRestConfig, manager.Options{
488522
Scheme: scheme,
523+
Cache: cache.Options{DefaultTransform: cache.TransformStripManagedFields()},
489524
Metrics: metricsserver.Options{BindAddress: "0"},
490525
})
491526
if err != nil {

go.mod

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module go.datum.net/compute
33
go 1.25.0
44

55
require (
6-
github.com/go-logr/logr v1.4.3
6+
github.com/KimMachineGun/automemlimit v0.7.5
77
github.com/google/go-cmp v0.7.0
88
github.com/karmada-io/api v1.15.0
99
github.com/onsi/ginkgo/v2 v2.27.2
@@ -39,6 +39,7 @@ require (
3939
github.com/felixge/httpsnoop v1.0.4 // indirect
4040
github.com/fsnotify/fsnotify v1.9.0 // indirect
4141
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
42+
github.com/go-logr/logr v1.4.3 // indirect
4243
github.com/go-logr/stdr v1.2.2 // indirect
4344
github.com/go-logr/zapr v1.3.0 // indirect
4445
github.com/go-openapi/jsonpointer v0.21.1 // indirect
@@ -58,6 +59,7 @@ require (
5859
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
5960
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
6061
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
62+
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
6163
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
6264
github.com/prometheus/client_model v0.6.2 // indirect
6365
github.com/prometheus/common v0.66.1 // indirect

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
22
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
3+
github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43DwZrJqVwyveTk=
4+
github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM=
35
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
46
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
57
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
@@ -107,6 +109,8 @@ github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns
107109
github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
108110
github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
109111
github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
112+
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
113+
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
110114
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
111115
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
112116
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

0 commit comments

Comments
 (0)