lib/storage: use lrucache to implement tagFilters loops cache

The tagFilters loop cache is per-indexDB which means that currently
there are two instances, one for idbCurr and one for idbPrev. When the
partition index (#8134) is released, there will be as many instances of
this cache as there will be partitions.

The cache is implemented using workingsetcache. Which occupies at least
30MB even when unused. Given that only the latest indexDB is used most
of the time, a lot of memory can be wasted.

Therefore the cache implementation is changed to lrucache because it
does not consume memory when it is unused and also has timeout-based
eviction.

This is a follow-up for 4cd727a
(VictoriaMetrics/VictoriaMetrics#10072).

Author: rtm0

lib/storage/index_db.go

@@ -31,7 +31,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/slicesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/uint64set"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/workingsetcache"
 )
 
 const (
@@ -134,7 +133,7 @@ type indexDB struct {
 
 	// Cache for (date, tagFilter) -> loopsCount, which is used for reducing
 	// the amount of work when matching a set of filters.
-	loopsPerDateTagFilterCache *workingsetcache.Cache
+	loopsPerDateTagFilterCache *lrucache.Cache
 
 	// dateMetricIDCache is (date, metricID) cache that is used to speed up the
 	// data ingestion by storing the is.hasDateMetricID() search results in
@@ -158,6 +157,10 @@ func getTagFiltersCacheSize() uint64 {
 	return maxTagFiltersCacheSize
 }
 
+func getTagFiltersLoopsCacheSize() uint64 {
+	return uint64(float64(memory.Allowed()) / 128)
+}
+
 // mustOpenIndexDB opens index db from the given path.
 //
 // The last segment of the path should contain unique hex value which
@@ -183,7 +186,7 @@ func mustOpenIndexDB(path string, s *Storage, isReadOnly *atomic.Bool, noRegiste
 		minMissingTimestampByKey:   make(map[string]int64),
 		tagFiltersToMetricIDsCache: tfssCache,
 		s:                          s,
-		loopsPerDateTagFilterCache: workingsetcache.New(memory.Allowed() / 128),
+		loopsPerDateTagFilterCache: lrucache.NewCache(getTagFiltersLoopsCacheSize),
 		dateMetricIDCache:          newDateMetricIDCache(),
 	}
 	db.noRegisterNewSeries.Store(noRegisterNewSeries)
@@ -298,13 +301,13 @@ func (db *indexDB) decRef() {
 	tbPath := db.tb.Path()
 	db.tb.MustClose()
 	db.tb = nil
+	db.s = nil
 
 	// Free space occupied by caches owned by db.
 	db.tagFiltersToMetricIDsCache.MustStop()
-	db.loopsPerDateTagFilterCache.Stop()
+	db.loopsPerDateTagFilterCache.MustStop()
 
 	db.tagFiltersToMetricIDsCache = nil
-	db.s = nil
 	db.loopsPerDateTagFilterCache = nil
 
 	if !db.mustDrop.Load() {
@@ -2974,25 +2977,32 @@ func (is *indexSearch) getLoopsCountAndTimestampForDateFilter(date uint64, tf *t
 	is.kb.B = appendDateTagFilterCacheKey(is.kb.B[:0], is.db.name, date, tf)
 	kb := kbPool.Get()
 	defer kbPool.Put(kb)
-	kb.B = is.db.loopsPerDateTagFilterCache.Get(kb.B[:0], is.kb.B)
-	if len(kb.B) != 3*8 {
+	e := is.db.loopsPerDateTagFilterCache.GetEntry(bytesutil.ToUnsafeString(is.kb.B))
+	if e == nil {
 		return 0, 0, 0
 	}
-	loopsCount := encoding.UnmarshalInt64(kb.B)
-	filterLoopsCount := encoding.UnmarshalInt64(kb.B[8:])
-	timestamp := encoding.UnmarshalUint64(kb.B[16:])
-	return loopsCount, filterLoopsCount, timestamp
+	v := e.(*tagFiltersLoops)
+	return v.loopsCount, v.filterLoopsCount, v.timestamp
+}
+
+type tagFiltersLoops struct {
+	loopsCount       int64
+	filterLoopsCount int64
+	timestamp        uint64
+}
+
+func (v *tagFiltersLoops) SizeBytes() uint64 {
+	return uint64(unsafe.Sizeof(*v))
 }
 
 func (is *indexSearch) storeLoopsCountForDateFilter(date uint64, tf *tagFilter, loopsCount, filterLoopsCount int64) {
-	currentTimestamp := fasttime.UnixTimestamp()
+	v := tagFiltersLoops{
+		loopsCount:       loopsCount,
+		filterLoopsCount: filterLoopsCount,
+		timestamp:        fasttime.UnixTimestamp(),
+	}
 	is.kb.B = appendDateTagFilterCacheKey(is.kb.B[:0], is.db.name, date, tf)
-	kb := kbPool.Get()
-	kb.B = encoding.MarshalInt64(kb.B[:0], loopsCount)
-	kb.B = encoding.MarshalInt64(kb.B, filterLoopsCount)
-	kb.B = encoding.MarshalUint64(kb.B, currentTimestamp)
-	is.db.loopsPerDateTagFilterCache.Set(is.kb.B, kb.B)
-	kbPool.Put(kb)
+	is.db.loopsPerDateTagFilterCache.PutEntry(string(is.kb.B), &v)
 }
 
 func appendDateTagFilterCacheKey(dst []byte, indexDBName string, date uint64, tf *tagFilter) []byte {