Skip to content

Commit 6446768

Browse files
committed
fix(loki.rules.kubernetes, mimir.rules.kubernetes): add timeout to ruler sync calls
Add a 30s context timeout to the ListRules calls in syncLoki and syncMimir to prevent the event loop goroutine from blocking indefinitely when the remote ruler is unreachable (e.g. due to a firewall blackhole). The shared RulerSyncTimeout constant is defined once in the common kubernetes package and reused by both components.
1 parent 54664b2 commit 6446768

4 files changed

Lines changed: 19 additions & 0 deletions

File tree

CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
# Changelog
22

3+
## Unreleased
4+
5+
### Bug Fixes 🐛
6+
7+
* **loki.rules.kubernetes, mimir.rules.kubernetes:** Add a 30s timeout to remote ruler sync (ListRules) calls to prevent the event loop from hanging indefinitely on transient network issues.
8+
39
## [1.15.0](https://github.com/grafana/alloy/compare/v1.14.0...v1.15.0) (2026-03-26)
410

511

internal/component/common/kubernetes/event.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package kubernetes
22

33
import (
4+
"time"
5+
46
"github.com/go-kit/log"
57
"github.com/grafana/alloy/internal/runtime/logging/level"
68
"k8s.io/client-go/tools/cache"
@@ -18,6 +20,11 @@ type EventType string
1820

1921
const (
2022
EventTypeResourceChanged EventType = "resource-changed"
23+
24+
// RulerSyncTimeout is the timeout applied to remote ruler API calls (e.g.
25+
// listing rule groups) to prevent the event loop from blocking indefinitely
26+
// on transient network issues.
27+
RulerSyncTimeout = 30 * time.Second
2128
)
2229

2330
type queuedEventHandler struct {

internal/component/loki/rules/kubernetes/events.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ func (c *Component) processEvent(ctx context.Context, e kubernetes.Event) error
8080
}
8181

8282
func (c *Component) syncLoki(ctx context.Context) error {
83+
ctx, cancel := context.WithTimeout(ctx, kubernetes.RulerSyncTimeout)
84+
defer cancel()
85+
8386
rulesByNamespace, err := c.lokiClient.ListRules(ctx, "")
8487
if err != nil {
8588
level.Error(c.log).Log("msg", "failed to list rules from loki", "err", err)

internal/component/mimir/rules/kubernetes/events.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ func (e *eventProcessor) enqueueSyncMimir() {
123123
}
124124

125125
func (e *eventProcessor) syncMimir(ctx context.Context) error {
126+
ctx, cancel := context.WithTimeout(ctx, kubernetes.RulerSyncTimeout)
127+
defer cancel()
128+
126129
rulesByNamespace, err := e.mimirClient.ListRules(ctx, "")
127130
if err != nil {
128131
level.Error(e.logger).Log("msg", "failed to list rules from mimir", "err", err)

0 commit comments

Comments
 (0)