fix: limit number of concurrent optimized compactions (#26319)

Limit number of concurrent optimized compactions so that level compactions do not get starved. Starved level compactions result in a sudden increase in disk usage.

Add [data] max-concurrent-optimized-compactions for configuring maximum number of concurrent optimized compactions. Default value is 1.

Co-authored-by: davidby-influx <[email protected]>
Co-authored-by: devanbenz <[email protected]>
Closes: #26315

Author: 3 people

tsdb/config.go

@@ -146,12 +146,17 @@ type Config struct {
 	// A value of 0 disables the limit.
 	MaxValuesPerTag int `toml:"max-values-per-tag"`
 
-	// MaxConcurrentCompactions is the maximum number of concurrent level and full compactions
+	// MaxConcurrentCompactions is the maximum number of concurrent level, full, and optimized compactions
 	// that can be running at one time across all shards.  Compactions scheduled to run when the
 	// limit is reached are blocked until a running compaction completes.  Snapshot compactions are
 	// not affected by this limit.  A value of 0 limits compactions to runtime.GOMAXPROCS(0).
 	MaxConcurrentCompactions int `toml:"max-concurrent-compactions"`
 
+	// MaxConcurrentOptimizedCompactions is the maximum number of concurrent optimized compactions
+	// that can be running across all shards. Optimized compactions scheduled to run when the limit
+	// is reached are aborted, saving them for a later compaction run.
+	MaxConcurrentOptimizedCompactions int `toml:"max-concurrent-optimized-compactions"`
+
 	// MaxIndexLogFileSize is the threshold, in bytes, when an index write-ahead log file will
 	// compact into an index file. Lower sizes will cause log files to be compacted more quickly
 	// and result in lower heap usage at the expense of write throughput. Higher sizes will

tsdb/engine.go

@@ -176,6 +176,7 @@ type EngineOptions struct {
 	CompactionPlannerCreator    CompactionPlannerCreator
 	CompactionLimiter           limiter.Fixed
 	CompactionThroughputLimiter limiter.Rate
+	OptimizedCompactionLimiter  limiter.Fixed
 	WALEnabled                  bool
 	MonitorDisabled             bool
 

tsdb/engine/tsm1/compact.go

@@ -270,14 +270,18 @@ func (c *DefaultPlanner) generationsFullyCompacted(gens tsmGenerations) (bool, s
 			aggressivePointsPerBlockCount := 0
 			filesUnderMaxTsmSizeCount := 0
 			for _, tsmFile := range gens[0].files {
-				if c.FileStore.BlockCount(tsmFile.Path, 1) >= c.aggressiveCompactionPointsPerBlock {
+				if c.FileStore.BlockCount(tsmFile.Path, 1) >= c.GetAggressiveCompactionPointsPerBlock() {
 					aggressivePointsPerBlockCount++
 				}
 				if tsmFile.Size < tsdb.MaxTSMFileSize {
 					filesUnderMaxTsmSizeCount++
 				}
 			}
 
+			if aggressivePointsPerBlockCount == len(gens[0].files) {
+				return true, "fully compacted because all files are at aggressivePointsPerBlock"
+			}
+
 			if filesUnderMaxTsmSizeCount > 1 && aggressivePointsPerBlockCount < len(gens[0].files) {
 				return false, tsdb.SingleGenerationReasonText
 			}