Fix Prometheus gauge collision for cache_size metric (#9729)

## What Changed

The replication progress cache was reusing
`MutableStateCacheTypeTagValue` as its `cache_type` metrics tag, causing
its `cache_size` gauge to overwrite the mutable state cache's gauge via
Prometheus last-write-wins semantics.

This made `cache_size{cache_type="mutablestate"}` always report `128000`
(the replication progress cache's default size) instead of the actual
mutable state cache capacity — particularly misleading when
`cacheSizeBasedLimit: true` is configured.

## Fix

- Add a new `ReplicationProgressCacheTypeTagValue =
"replication_progress"` constant in `common/metrics/metric_defs.go`
- Use it in `service/history/replication/progress_cache.go` instead of
`MutableStateCacheTypeTagValue`
- Add a unit test verifying the progress cache uses a distinct
`cache_type` tag value

Both caches now report their `cache_size` independently via distinct
`cache_type` label values.

## Why

This is a one-line behavioral change (plus the new constant and test).
It only affects metric label values — no logic, API surface, or
persistence changes.

Fixes #9600

Co-authored-by: Jiechen Zhong <jiechen.zhong@temporal.io>

Author: brucearctor

common/metrics/metric_defs.go

@@ -51,6 +51,7 @@ const (
 	ReactivationSignalDedupCacheTypeTagValue          = "reactivation_signal_dedup"
 	RoutingInfoCacheTypeTagValue                      = "routing_info"
 	NexusEndpointRegistryReadThroughCacheTypeTagValue = "nexus_endpoint_registry_readthrough"
+	ReplicationProgressCacheTypeTagValue              = "replication_progress"
 
 	InvalidHistoryURITagValue    = "invalid_history_uri"
 	InvalidVisibilityURITagValue = "invalid_visibility_uri"

service/history/replication/progress_cache.go

@@ -58,7 +58,7 @@ func NewProgressCache(
 		TTL: config.ReplicationProgressCacheTTL(),
 	}
 	return &progressCacheImpl{
-		cache: cache.NewWithMetrics(maxSize, opts, handler.WithTags(metrics.CacheTypeTag(metrics.MutableStateCacheTypeTagValue))),
+		cache: cache.NewWithMetrics(maxSize, opts, handler.WithTags(metrics.CacheTypeTag(metrics.ReplicationProgressCacheTypeTagValue))),
 	}
 }
 

service/history/replication/progress_cache_test.go

@@ -10,6 +10,7 @@ import (
 	historyspb "go.temporal.io/server/api/history/v1"
 	persistencespb "go.temporal.io/server/api/persistence/v1"
 	"go.temporal.io/server/common/metrics"
+	"go.temporal.io/server/common/metrics/metricstest"
 	"go.temporal.io/server/common/persistence/versionhistory"
 	"go.temporal.io/server/common/testing/protorequire"
 	"go.temporal.io/server/service/history/shard"
@@ -161,3 +162,40 @@ func (s *progressCacheSuite) TestProgressCache() {
 	cachedProgress = s.progressCache.Get(s.runID, targetClusterID)
 	s.DeepEqual(expected3, cachedProgress)
 }
+
+func TestProgressCacheUsesDistinctMetricsTag(t *testing.T) {
+	// Verify that the progress cache uses its own cache_type tag value
+	// ("replication_progress") rather than the mutable state cache's tag value
+	// ("mutablestate"), preventing Prometheus gauge collision. See #9600.
+	ctrl := gomock.NewController(t)
+	mockShard := shard.NewTestContext(
+		ctrl,
+		&persistencespb.ShardInfo{
+			ShardId: 0,
+			RangeId: 1,
+		},
+		tests.NewDynamicConfig(),
+	)
+	defer ctrl.Finish()
+	defer mockShard.StopForTest()
+
+	metricsHandler := metricstest.NewCaptureHandler()
+	capture := metricsHandler.StartCapture()
+
+	NewProgressCache(mockShard.GetConfig(), mockShard.GetLogger(), metricsHandler)
+
+	snapshot := capture.Snapshot()
+	cacheSizeRecordings := snapshot[metrics.CacheSize.Name()]
+	require.NotEmpty(t, cacheSizeRecordings, "expected cache_size metric to be recorded")
+
+	// Verify the cache_type tag is "replication_progress", not "mutablestate"
+	found := false
+	for _, recording := range cacheSizeRecordings {
+		if tag, ok := recording.Tags[metrics.CacheTypeTagName]; ok {
+			require.Equal(t, metrics.ReplicationProgressCacheTypeTagValue, tag,
+				"progress cache should use ReplicationProgressCacheTypeTagValue, not MutableStateCacheTypeTagValue")
+			found = true
+		}
+	}
+	require.True(t, found, "expected cache_size metric with cache_type tag")
+}