ruler: Add flag to configure the queue for pending Alertmanager notifications

In the [prometheus package](https://github.com/prometheus/prometheus/blob/7789ef
27c838a567edd65b5fb767d4a35cab91ac/cmd/prometheus/main.go#L532) we are already a
ble to configure the capacity size. The fixed value could lead to issues
sending alerts when evaluating a huge amount of alerts in one ruler
instance.

Signed-off-by: roth-wine <[email protected]>

Author: roth-wine

CHANGELOG.md

@@ -14,6 +14,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#8225](https://github.com/thanos-io/thanos/pull/8225) tools: Extend bucket ls options.
 
 ### Added
+- [#8234](https://github.com/thanos-io/thanos/pull/8234) ruler: Add flag to configure the queue for pending Alertmanager notifications
 
 ### Changed
 

cmd/thanos/rule.go

@@ -109,6 +109,7 @@ type ruleConfig struct {
 	ignoredLabelNames  []string
 	storeRateLimits    store.SeriesSelectLimits
 	ruleConcurrentEval int64
+	ruleQueueCapacity  int
 
 	extendedFunctionsEnabled bool
 }
@@ -160,6 +161,7 @@ func registerRule(app *extkingpin.App) {
 	cmd.Flag("restore-ignored-label", "Label names to be ignored when restoring alerts from the remote storage. This is only used in stateless mode.").
 		StringsVar(&conf.ignoredLabelNames)
 	cmd.Flag("rule-concurrent-evaluation", "How many rules can be evaluated concurrently. Default is 1.").Default("1").Int64Var(&conf.ruleConcurrentEval)
+	cmd.Flag("rule-queue-capacity", "The capacity of the queue for pending Alertmanager notifications. Default is 10000").Default("10000").IntVar(&conf.ruleQueueCapacity)
 
 	cmd.Flag("grpc-query-endpoint", "Addresses of Thanos gRPC query API servers (repeatable). The scheme may be prefixed with 'dns+' or 'dnssrv+' to detect Thanos API servers through respective DNS lookups.").
 		PlaceHolder("<endpoint>").StringsVar(&conf.grpcQueryEndpoints)
@@ -572,7 +574,7 @@ func runRule(
 
 	var (
 		ruleMgr *thanosrules.Manager
-		alertQ  = alert.NewQueue(logger, reg, 10000, 100, labelsTSDBToProm(conf.lset), conf.alertmgr.alertExcludeLabels, alertRelabelConfigs)
+		alertQ  = alert.NewQueue(logger, reg, conf.ruleQueueCapacity, 100, labelsTSDBToProm(conf.lset), conf.alertmgr.alertExcludeLabels, alertRelabelConfigs)
 	)
 	{
 		if conf.extendedFunctionsEnabled {

docs/components/rule.md

@@ -477,6 +477,9 @@ Flags:
                                  them.
       --rule-query-offset=0s     The default rule group query_offset duration to
                                  use.
+      --rule-queue-capacity=10000
+                                 The capacity of the queue for pending
+                                 Alertmanager notifications. Default is 10000
       --shipper.meta-file-name="thanos.shipper.json"
                                  the file to store shipper metadata in
       --shipper.upload-compacted