diff --git a/notify/notify.go b/notify/notify.go index 8fa85c0a21..3973e7876b 100644 --- a/notify/notify.go +++ b/notify/notify.go @@ -828,7 +828,7 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*types.A } for { - i++ + // Always check the context first to not notify again. select { case <-ctx.Done(): @@ -852,6 +852,7 @@ func (r RetryStage) exec(ctx context.Context, l *slog.Logger, alerts ...*types.A case <-tick.C: now := time.Now() retry, err := r.integration.Notify(ctx, sent...) + i++ dur := time.Since(now) r.metrics.notificationLatencySeconds.WithLabelValues(r.labelValues...).Observe(dur.Seconds()) r.metrics.numNotificationRequestsTotal.WithLabelValues(r.labelValues...).Inc() diff --git a/notify/notify_test.go b/notify/notify_test.go index 9d2fad5a17..a22d36f8ff 100644 --- a/notify/notify_test.go +++ b/notify/notify_test.go @@ -499,6 +499,7 @@ func TestRetryStageWithContextCanceled(t *testing.T) { counter := r.metrics.numTotalFailedNotifications require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues(r.integration.Name(), ContextCanceledReason.String())))) + require.Contains(t, err.Error(), "notify retry canceled after 1 attempts: context canceled") require.Error(t, err) require.NotNil(t, resctx)