fix: reduce API test memory consumption from 8.26GB to 1.57GB (#8263)

pkg/api/serve_test.go

@@ -113,6 +113,11 @@ func setupHandler(t testing.TB) (http.Handler, *dependencies) {
 	// Add endpoint so that 'IsAdvancedAuth' will be in effect
 	viper.Set("auth.api.endpoint", config.DefaultListenAddress)
 
+	// Set minimal cache sizes for testing to reduce memory consumption
+	// Default production values are too large for test environments
+	viper.Set("committed.local_cache.size_bytes", 8*1024*1024)          // 8MB instead of 1GB
+	viper.Set("committed.sstable.memory.cache_size_bytes", 2*1024*1024) // 2MB instead of 400MB
+
 	collector := &memCollector{}
 	cfg := &configfactory.ConfigWithAuth{}
 	baseCfg, err := config.NewConfig("", cfg)

pkg/pyramid/eviction.go

@@ -25,8 +25,22 @@ const (
 func newRistrettoEviction(capacity int64, evict func(rPath params.RelativePath, cost int64)) (params.Eviction, error) {
 	re := &ristrettoEviction{evictCallback: evict}
 
+	// Scale numCounters based on capacity to avoid excessive memory usage for small caches
+	// Default 10M counters is too much for test environments with small cache sizes
+	var numCountersToUse int64 = numCounters
+	if capacity < 100*1024*1024 { // If capacity < 100MB
+		// Use a more reasonable ratio: ~100 counters per 1MB of capacity
+		numCountersToUse = capacity / (1024 * 10) // ~100 counters per 10KB
+		if numCountersToUse < 1000 {
+			numCountersToUse = 1000 // Minimum threshold
+		}
+		if numCountersToUse > numCounters {
+			numCountersToUse = numCounters // Don't exceed default for large caches
+		}
+	}
+
 	cache, err := ristretto.NewCache(&ristretto.Config{
-		NumCounters: numCounters,
+		NumCounters: numCountersToUse,
 		MaxCost:     capacity,
 		BufferItems: bufferItems,
 		OnEvict:     re.onEvict,