TSDB head too far in future improvements (#11961)

#### What this PR does

#### Which issue(s) this PR fixes or relates to

Related to: #11928

#### Checklist

- [x] Tests updated.
- [ ] Documentation added.
- [x] `CHANGELOG.md` updated - the order of entries should be
`[CHANGE]`, `[FEATURE]`, `[ENHANCEMENT]`, `[BUGFIX]`. If changelog entry
is not needed, please add the `changelog-not-needed` label to the PR.
- [ ]
[`about-versioning.md`](https://github.com/grafana/mimir/blob/main/docs/sources/mimir/configure/about-versioning.md)
updated with experimental features.

Author: juliusmh

CHANGELOG.md

@@ -38,6 +38,7 @@
 * [FEATURE] You can configure Mimir to export traces in OTLP exposition format through the standard `OTEL_` environment variables. #11618
 * [FEATURE] distributor: Allow configuring tenant-specific HA tracker failover timeouts. #11774
 * [FEATURE] OTLP: Add experimental support for promoting OTel scope metadata (name, version, schema URL, attributes) to metric labels, prefixed with `otel_scope_`. Enable via the `-distributor.otel-promote-scope-metadata` flag. #11795
+* [ENHANCEMENT] Ingester: Display user grace interval in the tenant list obtained through the `/ingester/tenants` endpoint. #11961
 * [ENHANCEMENT] Dashboards: Add "Influx write requests" row to Writes Dashboard. #11731
 * [ENHANCEMENT] Mixin: Add `MimirHighVolumeLevel1BlocksQueried` alert that fires when level 1 blocks are queried for more than 6 hours, indicating potential compactor performance issues. #11803
 * [ENHANCEMENT] Querier: Make the maximum series limit for cardinality API requests configurable on a per-tenant basis with the `cardinality_analysis_max_results` option. #11456
@@ -191,6 +192,7 @@
 
 ### Documentation
 
+* [ENHANCEMENT] Update the `MimirIngestedDataTooFarInTheFuture` runbook with a note about false positives and the endpoint to flush TSDB blocks by user. #11961
 * [ENHANCEMENT] Update Thanos to Mimir migration guide with a tip to add the `__tenant_id__` label. #11584
 
 ### Tools

docs/sources/mimir/manage/mimir-runbooks/_index.md

@@ -1416,12 +1416,18 @@ How it **works**:
 
 - The metric exported by the ingester computes the maximum timestamp from all TSDBs open in the ingester.
 - The alert checks the metric and fires if the maximum timestamp is more than 1h in the future.
+- This alert doesn't respect the per-user creation grace period.
 
 How to **investigate**
 
-- Find the tenant with a bad sample on an affected ingester's tenants list (obtained via the `/ingester/tenants` endpoint), where a warning "TSDB Head max timestamp too far in the future" is displayed.
-- Flush the tenant's data to blocks storage.
-- Remove the tenant's directory on disk and the restart ingester.
+- Find an affected ingester pod and connect to its http API. For example, using `kubectl port-forward <pod> 8080:80`.
+- Find the tenant with a bad sample on the ingester's tenants list, which you can get through the `/ingester/tenants` endpoint. The sample should display the warning "TSDB Head max timestamp too far in the future".
+- If there are no warnings, the sample is likely within the tenant's grace interval and the alert is a false positive.
+
+If it's not a false positive:
+
+- Flush the tenant's data to blocks storage through the `/ingester/flush?wait=true&tenant=foo` endpoint.
+- Remove the tenant's directory on disk and restart the ingester.
 
 ### MimirStoreGatewayTooManyFailedOperations
 

pkg/ingester/tenants.gohtml

@@ -15,6 +15,7 @@
         <th>Blocks</th>
         <th>Head MinT</th>
         <th>Head MaxT</th>
+        <th>Grace interval</th>
         <th>Warning</th>
     </tr>
     </thead>
@@ -25,6 +26,7 @@
             <td>{{.Blocks}}</td>
             <td>{{.MinTime}}</td>
             <td>{{.MaxTime}}</td>
+            <td>(-{{.PastGracePeriod}}, +{{.FutureGracePeriod}})</td>
             <td>{{.Warning}}</td>
         </tr>
     {{ end }}

pkg/ingester/tenants_http.go

@@ -9,6 +9,7 @@ import (
 	"math"
 	"net/http"
 	"slices"
+	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -28,6 +29,12 @@ type tenantStats struct {
 	MinTime string
 	MaxTime string
 
+	// PastGracePeriod defines how far in the past we accept samples.
+	// This value includes OutOfOrderTimeWindow.
+	PastGracePeriod time.Duration
+	// FutureGracePeriod defines how far into the future we accept samples.
+	FutureGracePeriod time.Duration
+
 	Warning string
 }
 
@@ -79,21 +86,29 @@ func (i *Ingester) TenantsHandler(w http.ResponseWriter, req *http.Request) {
 			continue
 		}
 
+		var warnings []string
 		s := tenantStats{}
 		s.Tenant = t
 		s.Blocks = len(db.Blocks())
+		s.PastGracePeriod = i.limits.PastGracePeriod(t) + i.limits.OutOfOrderTimeWindow(t)
+		s.FutureGracePeriod = i.limits.CreationGracePeriod(t)
+
 		minMillis := db.Head().MinTime()
 		s.MinTime = formatMillisTime(db.Head().MinTime())
 		maxMillis := db.Head().MaxTime()
 		s.MaxTime = formatMillisTime(maxMillis)
 
-		if maxMillis-nowMillis > i.limits.CreationGracePeriod(t).Milliseconds() {
-			s.Warning = "TSDB Head max timestamp too far in the future"
+		if delta := maxMillis - nowMillis; delta > s.FutureGracePeriod.Milliseconds() {
+			deltaDuration := time.Duration(delta) * time.Millisecond
+			warning := fmt.Sprintf("TSDB Head max timestamp too far in the future: %v", deltaDuration)
+			warnings = append(warnings, warning)
 		}
 		if i.limits.PastGracePeriod(t) > 0 && nowMillis-minMillis > (i.limits.PastGracePeriod(t)+i.limits.OutOfOrderTimeWindow(t)).Milliseconds() {
-			s.Warning = "TSDB Head min timestamp too far in the past"
+			warnings = append(warnings, "TSDB Head min timestamp too far in the past")
 		}
 
+		s.Warning = strings.Join(warnings, ", ")
+
 		tss = append(tss, s)
 	}
 

pkg/ingester/tenants_http_test.go

@@ -44,6 +44,8 @@ func TestIngester_TenantsHandlers(t *testing.T) {
 		require.Equal(t, http.StatusOK, rec.Code)
 		// Check if link to user's TSDB was generated
 		require.Contains(t, rec.Body.String(), fmt.Sprintf(`<a href="tsdb/%s">%s</a>`, userID, userID))
+		// Check if grace interval is present
+		require.Contains(t, rec.Body.String(), `<td>(-0s, +10m0s)</td>`)
 	})
 
 	t.Run("tenant TSDB for valid tenant", func(t *testing.T) {