Fix postings cache size estimation and add postings count to spans (#980)

* Fix postings cache size estimation and add postings count to spans

The size.Of() already returns recursive size including postings cloner,
so we don't need to add EstimateSize(). Instead, track the actual number
of postings separately and add it to spans for better observability.

Previously we incorrectly estimated postings count as size/8, but the
size includes more than just postings data.

* Remove unnecessary casting

* Use plain int

* Fix tests

Author: dimitarvdimitrov

tsdb/index/postings.go

@@ -941,9 +941,8 @@ func (c *PostingsCloner) Clone() Postings {
 	return newListPostings(c.ids...)
 }
 
-// EstimateSize returns an estimate of the size of the PostingsCloner.
-func (c *PostingsCloner) EstimateSize() int64 {
-	return int64(len(c.ids) * 8)
+func (c *PostingsCloner) NumPostings() int {
+	return len(c.ids)
 }
 
 // FindIntersectingPostings checks the intersection of p and candidates[i] for each i in candidates,

tsdb/postings_for_matchers_cache.go

@@ -451,11 +451,12 @@ func (c *PostingsForMatchersCache) postingsForMatchersPromise(ctx context.Contex
 	promise.callersCtxTracker.Close()
 
 	sizeBytes := int64(len(key) + size.Of(promise))
+	numPostings := 0
 	if promise.cloner != nil {
-		sizeBytes += promise.cloner.EstimateSize()
+		numPostings = promise.cloner.NumPostings()
 	}
 
-	c.onPromiseExecutionDone(ctx, key, promise.evaluationCompletedAt, sizeBytes, promise.err)
+	c.onPromiseExecutionDone(ctx, key, promise.evaluationCompletedAt, sizeBytes, numPostings, promise.err)
 	return promise.result
 }
 
@@ -563,7 +564,7 @@ func (c *PostingsForMatchersCache) evictHead(now time.Time) (reason int) {
 // onPromiseExecutionDone must be called once the execution of PostingsForMatchers promise has done.
 // The input err contains details about any error that could have occurred when executing it.
 // The input ts is the function call time.
-func (c *PostingsForMatchersCache) onPromiseExecutionDone(ctx context.Context, key string, completedAt time.Time, sizeBytes int64, err error) {
+func (c *PostingsForMatchersCache) onPromiseExecutionDone(ctx context.Context, key string, completedAt time.Time, sizeBytes int64, numPostings int, err error) {
 	span := trace.SpanFromContext(ctx)
 
 	// Call the registered hook, if any. It's used only for testing purposes.
@@ -608,6 +609,7 @@ func (c *PostingsForMatchersCache) onPromiseExecutionDone(ctx context.Context, k
 		trace.WithAttributes(
 			attribute.Stringer("evaluation completed at", completedAt),
 			attribute.Int64("size in bytes", sizeBytes),
+			attribute.Int("num postings", numPostings),
 			attribute.Int64("cached bytes", lastCachedBytes),
 		),
 		trace.WithAttributes(c.additionalAttributes...),

tsdb/postings_for_matchers_cache_test.go

@@ -601,7 +601,7 @@ func TestPostingsForMatchersCache(t *testing.T) {
 				t.Run(fmt.Sprintf("shared=%t, invalidation=%t", shared, invalidation), func(t *testing.T) {
 					const (
 						maxItems         = 100 // Never hit it.
-						maxBytes         = 2400
+						maxBytes         = 1400
 						numMatchers      = 5
 						postingsListSize = 30 // 8 bytes per posting ref, so 30 x 8 = 240 bytes.
 					)
@@ -676,9 +676,9 @@ func TestPostingsForMatchersCache(t *testing.T) {
 					expectedBytes := 0
 					expectedEntries := 4
 					if shared {
-						expectedBytes = 2808
+						expectedBytes = 1848
 					} else {
-						expectedBytes = 2692
+						expectedBytes = 1732
 					}
 
 					require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(`
@@ -855,7 +855,7 @@ func TestPostingsForMatchersCache(t *testing.T) {
 		require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
 			# HELP postings_for_matchers_cache_bytes_total Total number of bytes in the PostingsForMatchers cache.
 			# TYPE postings_for_matchers_cache_bytes_total gauge
-			postings_for_matchers_cache_bytes_total 230
+			postings_for_matchers_cache_bytes_total 206
 
 			# HELP postings_for_matchers_cache_entries_total Total number of entries in the PostingsForMatchers cache.
 			# TYPE postings_for_matchers_cache_entries_total gauge
@@ -965,7 +965,7 @@ func TestPostingsForMatchersCache(t *testing.T) {
 		require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
 			# HELP postings_for_matchers_cache_bytes_total Total number of bytes in the PostingsForMatchers cache.
 			# TYPE postings_for_matchers_cache_bytes_total gauge
-			postings_for_matchers_cache_bytes_total 230
+			postings_for_matchers_cache_bytes_total 206
 
 			# HELP postings_for_matchers_cache_entries_total Total number of entries in the PostingsForMatchers cache.
 			# TYPE postings_for_matchers_cache_entries_total gauge