fix: cache_item_count metric overcounting (openfga#2950)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: justincoh

CHANGELOG.md

@@ -13,6 +13,7 @@ Try to keep listed changes to a concise bulleted list of simple explanations of
 ### Fixed
 - Fixed a few bugs. Two potential index out of bounds scenarios, and one cache of an invalid result. [#2942](https://github.com/openfga/openfga/pull/2942)
 - Fixed a race condition in check reducers causing non-deterministic nested handler execution due to canceled parent context. [#2947](https://github.com/openfga/openfga/pull/2947)
+- Fixed an issue where `cache_item_count` was incrementing on overwrites, causing the metric to steadily drift upward. [#2950](https://github.com/openfga/openfga/pull/2950)
 
 ## [1.11.6] - 2026-02-23
 ### Added

pkg/storage/cache.go

@@ -133,7 +133,6 @@ func NewInMemoryLRUCache[T any](opts ...InMemoryLRUCacheOpt[T]) (*InMemoryLRUCac
 			entityLabel = unspecifiedLabel
 		}
 
-		cacheItemCount.WithLabelValues(entityLabel).Dec()
 		cacheItemRemovedCount.WithLabelValues(entityLabel, reasonLabel).Inc()
 	})
 
@@ -163,12 +162,21 @@ func (i InMemoryLRUCache[T]) Set(key string, value T, ttl time.Duration) {
 	if ttl >= oneYear {
 		ttl = oneYear
 	}
+
+	if ttl < 0 {
+		return
+	}
+
+	// Ignore the boolean return here as we always pass cost=1 and items are always admitted
 	i.client.SetWithTTL(key, value, 1, ttl)
 
+	// Note: EstimatedSize is eventually consistent due to a shared lock in theine's maintenance routine.
+	// It shouldn't matter in practice, but it may lag behind a few entries.
+	cacheSizeFloat := float64(i.client.EstimatedSize())
 	if item, ok := any(value).(CacheItem); ok {
-		cacheItemCount.WithLabelValues(item.CacheEntityType()).Inc()
+		cacheItemCount.WithLabelValues(item.CacheEntityType()).Set(cacheSizeFloat)
 	} else {
-		cacheItemCount.WithLabelValues(unspecifiedLabel).Inc()
+		cacheItemCount.WithLabelValues(unspecifiedLabel).Set(cacheSizeFloat)
 	}
 }
 

pkg/storage/cache_test.go

@@ -6,11 +6,13 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/oklog/ulid/v2"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
 	"google.golang.org/protobuf/types/known/structpb"
@@ -421,6 +423,76 @@ func TestInMemoryCache(t *testing.T) {
 		require.NotEqual(t, "value", result)
 	})
 
+	t.Run("cache_item_count_doesnt_double_count_on_overwrite", func(t *testing.T) {
+		cache, err := NewInMemoryLRUCache[string]()
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			goleak.VerifyNone(t)
+		})
+		defer cache.Stop()
+		k := "key"
+
+		// Ensure metric is zero before we do anything
+		cacheItemCount.WithLabelValues(unspecifiedLabel).Set(float64(0))
+
+		cache.Set(k, "value1", time.Second)
+
+		// This .Wait() is needed as cache client.EstimatedSize() is eventually consistent
+		// due it its use of a shared mutex with theine's maintenance routine
+		cache.client.Wait()
+
+		cache.Set(k, "value2", time.Second)
+		before := testutil.ToFloat64(cacheItemCount.WithLabelValues(unspecifiedLabel))
+		cache.client.Wait()
+
+		cache.Set(k, "value3", time.Second)
+		after := testutil.ToFloat64(cacheItemCount.WithLabelValues(unspecifiedLabel))
+
+		// There should only be 1
+		require.InDelta(t, 0, after, 1)
+
+		// Should not have changed further
+		require.InDelta(t, 0, after-before, 0)
+	})
+
+	t.Run("cache_item_count_decrements_after_deletes", func(t *testing.T) {
+		cache, err := NewInMemoryLRUCache[string]()
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			goleak.VerifyNone(t)
+		})
+		defer cache.Stop()
+
+		// set 10 items in cache
+		for i := range 10 {
+			cache.Set(strconv.Itoa(i), "value"+strconv.Itoa(i), time.Second)
+		}
+		// Allow maintenance routine to finish
+		cache.client.Wait()
+
+		// Set once more to ensure metric has caught up
+		cache.Set("10", "value10", time.Second)
+
+		// this will return either 10 or 11 bc of race in EstimatedSize()
+		before := testutil.ToFloat64(cacheItemCount.WithLabelValues(unspecifiedLabel))
+
+		// delete all 11 items in cache
+		for i := range 11 {
+			cache.Delete(strconv.Itoa(i))
+		}
+		// Allow maintenance routine to finish
+		cache.client.Wait()
+
+		// The cache item count is only updated on Set()
+		cache.Set("10", "value10", time.Second)
+
+		// this will be either 0 or 1 after the Set call above
+		after := testutil.ToFloat64(cacheItemCount.WithLabelValues(unspecifiedLabel))
+
+		// expect before and after to differ by 9 elements, + or - 2 due to race
+		require.InDelta(t, 9, before-after, 2)
+	})
+
 	t.Run("stop_multiple_times", func(t *testing.T) {
 		cache, err := NewInMemoryLRUCache[string]()
 		require.NoError(t, err)