diff --git a/internal/component/common/kubernetes/event.go b/internal/component/common/kubernetes/event.go index 50fddf39e08..91a03a47d73 100644 --- a/internal/component/common/kubernetes/event.go +++ b/internal/component/common/kubernetes/event.go @@ -1,6 +1,8 @@ package kubernetes import ( + "time" + "github.com/go-kit/log" "github.com/grafana/alloy/internal/runtime/logging/level" "k8s.io/client-go/tools/cache" @@ -18,6 +20,11 @@ type EventType string const ( EventTypeResourceChanged EventType = "resource-changed" + + // RulerSyncTimeout is the timeout applied to remote ruler API calls (e.g. + // listing rule groups) to prevent the event loop from blocking indefinitely + // on transient network issues. + RulerSyncTimeout = 30 * time.Second ) type queuedEventHandler struct { diff --git a/internal/component/loki/rules/kubernetes/events.go b/internal/component/loki/rules/kubernetes/events.go index 0b1548ea54f..74915c0ba8f 100644 --- a/internal/component/loki/rules/kubernetes/events.go +++ b/internal/component/loki/rules/kubernetes/events.go @@ -80,6 +80,9 @@ func (c *Component) processEvent(ctx context.Context, e kubernetes.Event) error } func (c *Component) syncLoki(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, kubernetes.RulerSyncTimeout) + defer cancel() + rulesByNamespace, err := c.lokiClient.ListRules(ctx, "") if err != nil { level.Error(c.log).Log("msg", "failed to list rules from loki", "err", err) diff --git a/internal/component/mimir/rules/kubernetes/events.go b/internal/component/mimir/rules/kubernetes/events.go index 2d0e2a244f0..2e412344c12 100644 --- a/internal/component/mimir/rules/kubernetes/events.go +++ b/internal/component/mimir/rules/kubernetes/events.go @@ -123,6 +123,9 @@ func (e *eventProcessor) enqueueSyncMimir() { } func (e *eventProcessor) syncMimir(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, kubernetes.RulerSyncTimeout) + defer cancel() + rulesByNamespace, err := e.mimirClient.ListRules(ctx, "") if err != nil { level.Error(e.logger).Log("msg", "failed to list rules from mimir", "err", err)