Skip to content

fix: reduce API test memory consumption from 8.26GB to 1.57GB (#8263) #9097

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions pkg/api/serve_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,11 @@ func setupHandler(t testing.TB) (http.Handler, *dependencies) {
// Add endpoint so that 'IsAdvancedAuth' will be in effect
viper.Set("auth.api.endpoint", config.DefaultListenAddress)

// Set minimal cache sizes for testing to reduce memory consumption
// Default production values are too large for test environments
viper.Set("committed.local_cache.size_bytes", 8*1024*1024) // 8MB instead of 1GB
viper.Set("committed.sstable.memory.cache_size_bytes", 2*1024*1024) // 2MB instead of 400MB

collector := &memCollector{}
cfg := &configfactory.ConfigWithAuth{}
baseCfg, err := config.NewConfig("", cfg)
Expand Down
16 changes: 15 additions & 1 deletion pkg/pyramid/eviction.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,22 @@ const (
func newRistrettoEviction(capacity int64, evict func(rPath params.RelativePath, cost int64)) (params.Eviction, error) {
re := &ristrettoEviction{evictCallback: evict}

// Scale numCounters based on capacity to avoid excessive memory usage for small caches
// Default 10M counters is too much for test environments with small cache sizes
var numCountersToUse int64 = numCounters
if capacity < 100*1024*1024 { // If capacity < 100MB
// Use a more reasonable ratio: ~100 counters per 1MB of capacity
numCountersToUse = capacity / (1024 * 10) // ~100 counters per 10KB
if numCountersToUse < 1000 {
numCountersToUse = 1000 // Minimum threshold
}
if numCountersToUse > numCounters {
numCountersToUse = numCounters // Don't exceed default for large caches
}
}

cache, err := ristretto.NewCache(&ristretto.Config{
NumCounters: numCounters,
NumCounters: numCountersToUse,
MaxCost: capacity,
BufferItems: bufferItems,
OnEvict: re.onEvict,
Expand Down