diff --git a/kubernetes/controllers/servicelevelobjective_test.go b/kubernetes/controllers/servicelevelobjective_test.go index 07d308e58..526890745 100644 --- a/kubernetes/controllers/servicelevelobjective_test.go +++ b/kubernetes/controllers/servicelevelobjective_test.go @@ -155,28 +155,28 @@ func Test_makePrometheusRule(t *testing.T) { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`http_requests:burnrate5m{job="app",slo="http"} > (14 * (1-0.995)) and http_requests:burnrate1h{job="app",slo="http"} > (14 * (1-0.995))`), For: "2m", - Labels: map[string]string{"severity": "critical", "job": "app", "long": "1h", "slo": "http", "short": "5m", "team": "foo"}, + Labels: map[string]string{"severity": "critical", "job": "app", "long": "1h", "slo": "http", "short": "5m", "team": "foo", "exhaustion": "2d"}, Annotations: map[string]string{"description": "foo"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`http_requests:burnrate30m{job="app",slo="http"} > (7 * (1-0.995)) and http_requests:burnrate6h{job="app",slo="http"} > (7 * (1-0.995))`), + Expr: intstr.FromString(`http_requests:burnrate30m{job="app",slo="http"} > (5.6 * (1-0.995)) and http_requests:burnrate6h{job="app",slo="http"} > (5.6 * (1-0.995))`), For: "15m", - Labels: map[string]string{"severity": "critical", "job": "app", "long": "6h", "slo": "http", "short": "30m", "team": "foo"}, + Labels: map[string]string{"severity": "critical", "job": "app", "long": "6h", "slo": "http", "short": "30m", "team": "foo", "exhaustion": "5d"}, Annotations: map[string]string{"description": "foo"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`http_requests:burnrate2h{job="app",slo="http"} > (2 * (1-0.995)) and http_requests:burnrate1d{job="app",slo="http"} > (2 * (1-0.995))`), + Expr: intstr.FromString(`http_requests:burnrate2h{job="app",slo="http"} > (2.8 * (1-0.995)) and http_requests:burnrate1d{job="app",slo="http"} > (2.8 * (1-0.995))`), For: "1h", - Labels: map[string]string{"severity": "warning", "job": "app", "long": "1d", "slo": "http", "short": "2h", "team": "foo"}, + Labels: map[string]string{"severity": "warning", "job": "app", "long": "1d", "slo": "http", "short": "2h", "team": "foo", "exhaustion": "10d"}, Annotations: map[string]string{"description": "foo"}, }, { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`http_requests:burnrate6h{job="app",slo="http"} > (1 * (1-0.995)) and http_requests:burnrate4d{job="app",slo="http"} > (1 * (1-0.995))`), For: "3h", - Labels: map[string]string{"severity": "warning", "job": "app", "long": "4d", "slo": "http", "short": "6h", "team": "foo"}, + Labels: map[string]string{"severity": "warning", "job": "app", "long": "4d", "slo": "http", "short": "6h", "team": "foo", "exhaustion": "4w"}, Annotations: map[string]string{"description": "foo"}, }, }, @@ -268,6 +268,7 @@ func Test_makeConfigMap(t *testing.T) { > (14 * (1-0.995)) for: 2m labels: + exhaustion: 2d job: app long: 1h severity: critical @@ -277,10 +278,11 @@ func Test_makeConfigMap(t *testing.T) { - alert: ErrorBudgetBurn annotations: description: foo - expr: http_requests:burnrate30m{job="app",slo="http"} > (7 * (1-0.995)) and http_requests:burnrate6h{job="app",slo="http"} - > (7 * (1-0.995)) + expr: http_requests:burnrate30m{job="app",slo="http"} > (5.6 * (1-0.995)) and + http_requests:burnrate6h{job="app",slo="http"} > (5.6 * (1-0.995)) for: 15m labels: + exhaustion: 5d job: app long: 6h severity: critical @@ -290,10 +292,11 @@ func Test_makeConfigMap(t *testing.T) { - alert: ErrorBudgetBurn annotations: description: foo - expr: http_requests:burnrate2h{job="app",slo="http"} > (2 * (1-0.995)) and http_requests:burnrate1d{job="app",slo="http"} - > (2 * (1-0.995)) + expr: http_requests:burnrate2h{job="app",slo="http"} > (2.8 * (1-0.995)) and http_requests:burnrate1d{job="app",slo="http"} + > (2.8 * (1-0.995)) for: 1h labels: + exhaustion: 10d job: app long: 1d severity: warning @@ -307,6 +310,7 @@ func Test_makeConfigMap(t *testing.T) { > (1 * (1-0.995)) for: 3h labels: + exhaustion: 4w job: app long: 4d severity: warning diff --git a/main_test.go b/main_test.go index 2aa48e8da..d06fed73f 100644 --- a/main_test.go +++ b/main_test.go @@ -278,7 +278,7 @@ func TestAlertsMatchingObjectives(t *testing.T) { Severity: "critical", State: objectivesv1alpha1.Alert_inactive, For: durationpb.New(8 * time.Minute), - Factor: 7, + Factor: 5.6, Short: &objectivesv1alpha1.Burnrate{ Window: durationpb.New(15 * time.Minute), Current: -1, @@ -297,7 +297,7 @@ func TestAlertsMatchingObjectives(t *testing.T) { Severity: "warning", State: objectivesv1alpha1.Alert_inactive, For: durationpb.New(30 * time.Minute), - Factor: 2, + Factor: 2.8, Short: &objectivesv1alpha1.Burnrate{ Window: durationpb.New(time.Hour), Current: -1, @@ -373,7 +373,7 @@ func TestAlertsMatchingObjectives(t *testing.T) { Severity: "critical", State: objectivesv1alpha1.Alert_inactive, For: durationpb.New(8 * time.Minute), - Factor: 7, + Factor: 5.6, Short: &objectivesv1alpha1.Burnrate{ Window: durationpb.New(15 * time.Minute), Current: -1, @@ -390,7 +390,7 @@ func TestAlertsMatchingObjectives(t *testing.T) { Severity: "warning", State: objectivesv1alpha1.Alert_inactive, For: durationpb.New(30 * time.Minute), - Factor: 2, + Factor: 2.8, Short: &objectivesv1alpha1.Burnrate{ Window: durationpb.New(time.Hour), Current: -1, diff --git a/slo/rules.go b/slo/rules.go index 537caeb15..943acee5e 100644 --- a/slo/rules.go +++ b/slo/rules.go @@ -123,11 +123,12 @@ func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) { alertLabels["short"] = model.Duration(w.Short).String() alertLabels["long"] = model.Duration(w.Long).String() alertLabels["severity"] = string(w.Severity) + alertLabels["exhaustion"] = model.Duration(w.Exhaustion).String() r := monitoringv1.Rule{ Alert: o.AlertName(), // TODO: Use expr replacer - Expr: intstr.FromString(fmt.Sprintf("%s{%s} > (%.f * (1-%s)) and %s{%s} > (%.f * (1-%s))", + Expr: intstr.FromString(fmt.Sprintf("%s{%s} > (%v * (1-%s)) and %s{%s} > (%v * (1-%s))", o.BurnrateName(w.Short), alertMatchersString, w.Factor, @@ -208,11 +209,12 @@ func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) { alertLabels["short"] = model.Duration(w.Short).String() alertLabels["long"] = model.Duration(w.Long).String() alertLabels["severity"] = string(w.Severity) + alertLabels["exhaustion"] = model.Duration(w.Exhaustion).String() r := monitoringv1.Rule{ Alert: o.AlertName(), // TODO: Use expr replacer - Expr: intstr.FromString(fmt.Sprintf("%s{%s} > (%.f * (1-%s)) and %s{%s} > (%.f * (1-%s))", + Expr: intstr.FromString(fmt.Sprintf("%s{%s} > (%v * (1-%s)) and %s{%s} > (%v * (1-%s))", o.BurnrateName(w.Short), alertMatchersString, w.Factor, @@ -293,11 +295,12 @@ func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) { alertLabels["short"] = model.Duration(w.Short).String() alertLabels["long"] = model.Duration(w.Long).String() alertLabels["severity"] = string(w.Severity) + alertLabels["exhaustion"] = model.Duration(w.Exhaustion).String() r := monitoringv1.Rule{ Alert: o.AlertName(), // TODO: Use expr replacer - Expr: intstr.FromString(fmt.Sprintf("%s{%s} > (%.f * (1-%s)) and %s{%s} > (%.f * (1-%s))", + Expr: intstr.FromString(fmt.Sprintf("%s{%s} > (%v * (1-%s)) and %s{%s} > (%v * (1-%s))", o.BurnrateName(w.Short), alertMatchersString, w.Factor, @@ -378,11 +381,12 @@ func (o Objective) Burnrates() (monitoringv1.RuleGroup, error) { alertLabels["short"] = model.Duration(w.Short).String() alertLabels["long"] = model.Duration(w.Long).String() alertLabels["severity"] = string(w.Severity) + alertLabels["exhaustion"] = model.Duration(w.Exhaustion).String() r := monitoringv1.Rule{ Alert: o.AlertName(), // TODO: Use expr replacer - Expr: intstr.FromString(fmt.Sprintf("%s{%s} > (%.f * (1-%s)) and %s{%s} > (%.f * (1-%s))", + Expr: intstr.FromString(fmt.Sprintf("%s{%s} > (%v * (1-%s)) and %s{%s} > (%v * (1-%s))", o.BurnrateName(w.Short), alertMatchersString, w.Factor, @@ -1074,44 +1078,55 @@ const ( warning severity = "warning" ) +// Window represents a multi-window alert for a particular burn rate factor. type Window struct { - Severity severity - For time.Duration - Long time.Duration - Short time.Duration - Factor float64 + Severity severity + For time.Duration + Long time.Duration // Long represents the long window or the alerting window for a multi-window alert. + Short time.Duration // Short represents the short window or reset period of a multi-window alert. + Exhaustion time.Duration // Exhaustion specifies the time it takes to burn the whole error budget. + Factor float64 } +// Windows returns multi-window alerts across four burn rates, for a given SLO window. +// long and short rates are calculated based on the ratio for 28 days. +// Thus the alerts generated, would be based on the same burn rates, across different sloWindows +// which is why they are constant. +// +// The burn rate factors work best for 28 days or other windows that are multiples of 7. +// This can still be used with windows like 30d/90d, but might lead to uneven for/long/short. +// See Test_windows for examples. func Windows(sloWindow time.Duration) []Window { - // TODO: I'm still not sure if For, Long, Short should really be based on the 28 days ratio... - round := time.Minute // TODO: Change based on sloWindow - // long and short rates are calculated based on the ratio for 28 days. return []Window{{ - Severity: critical, - For: (sloWindow / (28 * 24 * (60 / 2))).Round(round), // 2m for 28d - half short - Long: (sloWindow / (28 * 24)).Round(round), // 1h for 28d - Short: (sloWindow / (28 * 24 * (60 / 5))).Round(round), // 5m for 28d - Factor: 14, // error budget burn: 50% within a day + Severity: critical, + For: (sloWindow / (28 * 24 * (60 / 2))).Round(round), // 2m for 28d - half short + Long: (sloWindow / (28 * 24)).Round(round), // 1h for 28d + Short: (sloWindow / (28 * 24 * (60 / 5))).Round(round), // 5m for 28d + Exhaustion: time.Duration(sloWindow / 14).Round(round), // error budget burn: 50% within a day / 100% within 2d for 28d + Factor: 14, // 50 / ((24/(24 * 28)) * 100) }, { - Severity: critical, - For: (sloWindow / (28 * 24 * (60 / 15))).Round(round), // 15m for 28d - half short - Long: (sloWindow / (28 * (24 / 6))).Round(round), // 6h for 28d - Short: (sloWindow / (28 * 24 * (60 / 30))).Round(round), // 30m for 28d - Factor: 7, // error budget burn: 20% within a day / 100% within 5 days + Severity: critical, + For: (sloWindow / (28 * 24 * (60 / 15))).Round(round), // 15m for 28d - half short + Long: (sloWindow / (28 * (24 / 6))).Round(round), // 6h for 28d + Short: (sloWindow / (28 * 24 * (60 / 30))).Round(round), // 30m for 28d + Exhaustion: time.Duration(float64(sloWindow) / 5.6).Round(round), // error budget burn: 20% within a day / 100% within 5 days for 28d + Factor: 5.6, // 20 / ((24/(24 * 28)) * 100) }, { - Severity: warning, - For: (sloWindow / (28 * 24)).Round(round), // 1h for 28d - half short - Long: (sloWindow / 28).Round(round), // 1d for 28d - Short: (sloWindow / (28 * (24 / 2))).Round(round), // 2h for 28d - Factor: 2, // error budget burn: 10% within a day / 100% within 10 days + Severity: warning, + For: (sloWindow / (28 * 24)).Round(round), // 1h for 28d - half short + Long: (sloWindow / 28).Round(round), // 1d for 28d + Short: (sloWindow / (28 * (24 / 2))).Round(round), // 2h for 28d + Exhaustion: time.Duration(float64(sloWindow) / 2.8).Round(round), // error budget burn: 10% within a day / 100% within 10 days for 28d + Factor: 2.8, // 10 / ((24/(24 * 28)) * 100) }, { - Severity: warning, - For: (sloWindow / (28 * (24 / 3))).Round(round), // 3h for 28d - half short - Long: (sloWindow / 7).Round(round), // 4d for 28d - Short: (sloWindow / (28 * (24 / 6))).Round(round), // 6h for 28d - Factor: 1, // error budget burn: 100% until the end of sloWindow + Severity: warning, + For: (sloWindow / (28 * (24 / 3))).Round(round), // 3h for 28d - half short + Long: (sloWindow / 7).Round(round), // 4d for 28d + Short: (sloWindow / (28 * (24 / 6))).Round(round), // 6h for 28d + Exhaustion: (sloWindow).Round(round), // error budget burn: 100% until the end of sloWindow + Factor: 1, // 100 / (((24 * 28)/(24 * 28)) * 100) }} } diff --git a/slo/rules_test.go b/slo/rules_test.go index 909562397..c056dfb1f 100644 --- a/slo/rules_test.go +++ b/slo/rules_test.go @@ -52,22 +52,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "2m", Expr: intstr.FromString(`http_requests:burnrate5m{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99)) and http_requests:burnrate1h{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "job": "thanos-receive-default", "long": "1h", "slo": "monitoring-http-errors", "short": "5m"}, + Labels: map[string]string{"severity": "critical", "job": "thanos-receive-default", "long": "1h", "slo": "monitoring-http-errors", "short": "5m", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", For: "15m", - Expr: intstr.FromString(`http_requests:burnrate30m{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99)) and http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "job": "thanos-receive-default", "long": "6h", "slo": "monitoring-http-errors", "short": "30m"}, + Expr: intstr.FromString(`http_requests:burnrate30m{job="thanos-receive-default",slo="monitoring-http-errors"} > (5.6 * (1-0.99)) and http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (5.6 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "job": "thanos-receive-default", "long": "6h", "slo": "monitoring-http-errors", "short": "30m", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h", - Expr: intstr.FromString(`http_requests:burnrate2h{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99)) and http_requests:burnrate1d{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "job": "thanos-receive-default", "long": "1d", "slo": "monitoring-http-errors", "short": "2h"}, + Expr: intstr.FromString(`http_requests:burnrate2h{job="thanos-receive-default",slo="monitoring-http-errors"} > (2.8 * (1-0.99)) and http_requests:burnrate1d{job="thanos-receive-default",slo="monitoring-http-errors"} > (2.8 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "job": "thanos-receive-default", "long": "1d", "slo": "monitoring-http-errors", "short": "2h", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", For: "3h", Expr: intstr.FromString(`http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99)) and http_requests:burnrate4d{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "job": "thanos-receive-default", "long": "4d", "slo": "monitoring-http-errors", "short": "6h"}, + Labels: map[string]string{"severity": "warning", "job": "thanos-receive-default", "long": "4d", "slo": "monitoring-http-errors", "short": "6h", "exhaustion": "4w"}, }}, }, }, { @@ -108,22 +108,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "2m", Expr: intstr.FromString(`http_requests:burnrate5m{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99)) and http_requests:burnrate1h{job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "slo": "monitoring-http-errors", "short": "5m"}, + Labels: map[string]string{"severity": "critical", "long": "1h", "slo": "monitoring-http-errors", "short": "5m", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", For: "15m", - Expr: intstr.FromString(`http_requests:burnrate30m{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99)) and http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-errors", "short": "30m"}, + Expr: intstr.FromString(`http_requests:burnrate30m{job="thanos-receive-default",slo="monitoring-http-errors"} > (5.6 * (1-0.99)) and http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (5.6 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-errors", "short": "30m", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h", - Expr: intstr.FromString(`http_requests:burnrate2h{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99)) and http_requests:burnrate1d{job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-errors", "short": "2h"}, + Expr: intstr.FromString(`http_requests:burnrate2h{job="thanos-receive-default",slo="monitoring-http-errors"} > (2.8 * (1-0.99)) and http_requests:burnrate1d{job="thanos-receive-default",slo="monitoring-http-errors"} > (2.8 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-errors", "short": "2h", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", For: "3h", Expr: intstr.FromString(`http_requests:burnrate6h{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99)) and http_requests:burnrate4d{job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-errors", "short": "6h"}, + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-errors", "short": "6h", "exhaustion": "4w"}, }}, }, }, { @@ -164,22 +164,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "2m", Expr: intstr.FromString(`http_requests:burnrate5m{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99)) and http_requests:burnrate1h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "monitoring-http-errors"}, + Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "monitoring-http-errors", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", For: "15m", - Expr: intstr.FromString(`http_requests:burnrate30m{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99)) and http_requests:burnrate6h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-errors", "short": "30m"}, + Expr: intstr.FromString(`http_requests:burnrate30m{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (5.6 * (1-0.99)) and http_requests:burnrate6h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (5.6 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-errors", "short": "30m", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h", - Expr: intstr.FromString(`http_requests:burnrate2h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99)) and http_requests:burnrate1d{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-errors", "short": "2h"}, + Expr: intstr.FromString(`http_requests:burnrate2h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (2.8 * (1-0.99)) and http_requests:burnrate1d{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (2.8 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-errors", "short": "2h", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", For: "3h", Expr: intstr.FromString(`http_requests:burnrate6h{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99)) and http_requests:burnrate4d{handler=~"/api.*",job="thanos-receive-default",slo="monitoring-http-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-errors", "short": "6h"}, + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-errors", "short": "6h", "exhaustion": "4w"}, }}, }, }, { @@ -220,22 +220,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`grpc_server_handled:burnrate5m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999)) and grpc_server_handled:burnrate1h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999))`), For: "2m", - Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "5m", "long": "1h"}, + Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "5m", "long": "1h", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999)) and grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999))`), + Expr: intstr.FromString(`grpc_server_handled:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (5.6 * (1-0.999)) and grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (5.6 * (1-0.999))`), For: "15m", - Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "30m", "long": "6h"}, + Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "30m", "long": "6h", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate2h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999)) and grpc_server_handled:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999))`), + Expr: intstr.FromString(`grpc_server_handled:burnrate2h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2.8 * (1-0.999)) and grpc_server_handled:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2.8 * (1-0.999))`), For: "1h", - Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "2h", "long": "1d"}, + Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "2h", "long": "1d", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999)) and grpc_server_handled:burnrate4d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999))`), For: "3h", - Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "6h", "long": "4d"}, + Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "job": "api", "slo": "monitoring-grpc-errors", "short": "6h", "long": "4d", "exhaustion": "4w"}, }}, }, }, { @@ -276,22 +276,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`grpc_server_handled:burnrate5m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999)) and grpc_server_handled:burnrate1h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (14 * (1-0.999))`), For: "2m", - Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "5m", "long": "1h"}, + Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "5m", "long": "1h", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999)) and grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (7 * (1-0.999))`), + Expr: intstr.FromString(`grpc_server_handled:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (5.6 * (1-0.999)) and grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (5.6 * (1-0.999))`), For: "15m", - Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "30m", "long": "6h"}, + Labels: map[string]string{"severity": "critical", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "30m", "long": "6h", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handled:burnrate2h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999)) and grpc_server_handled:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2 * (1-0.999))`), + Expr: intstr.FromString(`grpc_server_handled:burnrate2h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2.8 * (1-0.999)) and grpc_server_handled:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (2.8 * (1-0.999))`), For: "1h", - Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "2h", "long": "1d"}, + Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "2h", "long": "1d", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`grpc_server_handled:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999)) and grpc_server_handled:burnrate4d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-errors"} > (1 * (1-0.999))`), For: "3h", - Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "6h", "long": "4d"}, + Labels: map[string]string{"severity": "warning", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "slo": "monitoring-grpc-errors", "short": "6h", "long": "4d", "exhaustion": "4w"}, }}, }, }, { @@ -332,22 +332,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "2m", Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "5m"}, + Labels: map[string]string{"severity": "critical", "long": "1h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "5m", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", For: "15m", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "30m"}, + Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (5.6 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (5.6 * (1-0.995))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "30m", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "2h"}, + Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2.8 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2.8 * (1-0.995))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "2h", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", For: "3h", Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "6h"}, + Labels: map[string]string{"severity": "warning", "long": "4d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "6h", "exhaustion": "4w"}, }}, }, }, { @@ -388,22 +388,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "2m", Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "5m"}, + Labels: map[string]string{"severity": "critical", "long": "1h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "5m", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", For: "15m", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "30m"}, + Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (5.6 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (5.6 * (1-0.995))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "30m", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "2h"}, + Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2.8 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2.8 * (1-0.995))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "2h", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", For: "3h", Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "6h"}, + Labels: map[string]string{"severity": "warning", "long": "4d", "job": "metrics-service-thanos-receive-default", "slo": "monitoring-http-latency", "short": "6h", "exhaustion": "4w"}, }}, }, }, { @@ -444,22 +444,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "2m", Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "slo": "monitoring-http-latency", "short": "5m"}, + Labels: map[string]string{"severity": "critical", "long": "1h", "slo": "monitoring-http-latency", "short": "5m", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", For: "15m", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-latency", "short": "30m"}, + Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (5.6 * (1-0.995)) and http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (5.6 * (1-0.995))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "monitoring-http-latency", "short": "30m", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-latency", "short": "2h"}, + Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2.8 * (1-0.995)) and http_request_duration_seconds:burnrate1d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2.8 * (1-0.995))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "monitoring-http-latency", "short": "2h", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", For: "3h", Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-latency", "short": "6h"}, + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "monitoring-http-latency", "short": "6h", "exhaustion": "4w"}, }}, }, }, { @@ -500,22 +500,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`http_request_duration_seconds:burnrate5m{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995)) and http_request_duration_seconds:burnrate1h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (14 * (1-0.995))`), For: "2m", - Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "monitoring-http-latency"}, + Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "monitoring-http-latency", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995)) and http_request_duration_seconds:burnrate6h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (7 * (1-0.995))`), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate30m{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (5.6 * (1-0.995)) and http_request_duration_seconds:burnrate6h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (5.6 * (1-0.995))`), For: "15m", - Labels: map[string]string{"severity": "critical", "long": "6h", "short": "30m", "slo": "monitoring-http-latency"}, + Labels: map[string]string{"severity": "critical", "long": "6h", "short": "30m", "slo": "monitoring-http-latency", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995)) and http_request_duration_seconds:burnrate1d{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2 * (1-0.995))`), + Expr: intstr.FromString(`http_request_duration_seconds:burnrate2h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2.8 * (1-0.995)) and http_request_duration_seconds:burnrate1d{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (2.8 * (1-0.995))`), For: "1h", - Labels: map[string]string{"severity": "warning", "long": "1d", "short": "2h", "slo": "monitoring-http-latency"}, + Labels: map[string]string{"severity": "warning", "long": "1d", "short": "2h", "slo": "monitoring-http-latency", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`http_request_duration_seconds:burnrate6h{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995)) and http_request_duration_seconds:burnrate4d{handler=~"/api.*",job="metrics-service-thanos-receive-default",slo="monitoring-http-latency"} > (1 * (1-0.995))`), For: "3h", - Labels: map[string]string{"severity": "warning", "long": "4d", "short": "6h", "slo": "monitoring-http-latency"}, + Labels: map[string]string{"severity": "warning", "long": "4d", "short": "6h", "slo": "monitoring-http-latency", "exhaustion": "4w"}, }}, }, }, { @@ -556,22 +556,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995)) and grpc_server_handling_seconds:burnrate15m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995))`), For: "1m", - Labels: map[string]string{"severity": "critical", "long": "15m", "short": "1m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, + Labels: map[string]string{"severity": "critical", "long": "15m", "short": "1m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "12h"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate8m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995)) and grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995))`), + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate8m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (5.6 * (1-0.995)) and grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (5.6 * (1-0.995))`), For: "4m", - Labels: map[string]string{"severity": "critical", "long": "1h30m", "short": "8m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, + Labels: map[string]string{"severity": "critical", "long": "1h30m", "short": "8m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1d6h"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995)) and grpc_server_handling_seconds:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995))`), + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2.8 * (1-0.995)) and grpc_server_handling_seconds:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2.8 * (1-0.995))`), For: "15m", - Labels: map[string]string{"severity": "warning", "long": "6h", "short": "30m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, + Labels: map[string]string{"severity": "warning", "long": "6h", "short": "30m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "2d12h"}, }, { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995)) and grpc_server_handling_seconds:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995))`), For: "45m", - Labels: map[string]string{"severity": "warning", "long": "1d", "short": "1h30m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, + Labels: map[string]string{"severity": "warning", "long": "1d", "short": "1h30m", "slo": "monitoring-grpc-latency", "job": "api", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1w"}, }}, }, }, { @@ -612,22 +612,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995)) and grpc_server_handling_seconds:burnrate15m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (14 * (1-0.995))`), For: "1m", - Labels: map[string]string{"severity": "critical", "long": "15m", "short": "1m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, + Labels: map[string]string{"severity": "critical", "long": "15m", "short": "1m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "12h"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate8m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995)) and grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (7 * (1-0.995))`), + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate8m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (5.6 * (1-0.995)) and grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (5.6 * (1-0.995))`), For: "4m", - Labels: map[string]string{"severity": "critical", "long": "1h30m", "short": "8m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, + Labels: map[string]string{"severity": "critical", "long": "1h30m", "short": "8m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1d6h"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995)) and grpc_server_handling_seconds:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2 * (1-0.995))`), + Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2.8 * (1-0.995)) and grpc_server_handling_seconds:burnrate6h{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (2.8 * (1-0.995))`), For: "15m", - Labels: map[string]string{"severity": "warning", "long": "6h", "short": "30m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, + Labels: map[string]string{"severity": "warning", "long": "6h", "short": "30m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "2d12h"}, }, { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`grpc_server_handling_seconds:burnrate1h30m{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995)) and grpc_server_handling_seconds:burnrate1d{grpc_method="Write",grpc_service="conprof.WritableProfileStore",job="api",slo="monitoring-grpc-latency"} > (1 * (1-0.995))`), For: "45m", - Labels: map[string]string{"severity": "warning", "long": "1d", "short": "1h30m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore"}, + Labels: map[string]string{"severity": "warning", "long": "1d", "short": "1h30m", "slo": "monitoring-grpc-latency", "grpc_method": "Write", "grpc_service": "conprof.WritableProfileStore", "exhaustion": "1w"}, }}, }, }, { @@ -668,22 +668,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "1m", Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate30m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "30m", "slo": "monitoring-prometheus-operator-errors", "short": "3m"}, + Labels: map[string]string{"severity": "critical", "long": "30m", "slo": "monitoring-prometheus-operator-errors", "short": "3m", "exhaustion": "1d"}, }, { Alert: "ErrorBudgetBurn", For: "8m", - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate15m{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "3h", "slo": "monitoring-prometheus-operator-errors", "short": "15m"}, + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate15m{slo="monitoring-prometheus-operator-errors"} > (5.6 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (5.6 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "3h", "slo": "monitoring-prometheus-operator-errors", "short": "15m", "exhaustion": "2d12h"}, }, { Alert: "ErrorBudgetBurn", For: "30m", - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate1h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate12h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "12h", "slo": "monitoring-prometheus-operator-errors", "short": "1h"}, + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate1h{slo="monitoring-prometheus-operator-errors"} > (2.8 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate12h{slo="monitoring-prometheus-operator-errors"} > (2.8 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "12h", "slo": "monitoring-prometheus-operator-errors", "short": "1h", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h30m", Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate2d{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "2d", "slo": "monitoring-prometheus-operator-errors", "short": "3h"}, + Labels: map[string]string{"severity": "warning", "long": "2d", "slo": "monitoring-prometheus-operator-errors", "short": "3h", "exhaustion": "2w"}, }}, }, }, { @@ -724,22 +724,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "1m", Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate30m{slo="monitoring-prometheus-operator-errors"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "30m", "slo": "monitoring-prometheus-operator-errors", "short": "3m"}, + Labels: map[string]string{"severity": "critical", "long": "30m", "slo": "monitoring-prometheus-operator-errors", "short": "3m", "exhaustion": "1d"}, }, { Alert: "ErrorBudgetBurn", For: "8m", - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate15m{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "3h", "slo": "monitoring-prometheus-operator-errors", "short": "15m"}, + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate15m{slo="monitoring-prometheus-operator-errors"} > (5.6 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (5.6 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "3h", "slo": "monitoring-prometheus-operator-errors", "short": "15m", "exhaustion": "2d12h"}, }, { Alert: "ErrorBudgetBurn", For: "30m", - Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate1h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate12h{slo="monitoring-prometheus-operator-errors"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "12h", "slo": "monitoring-prometheus-operator-errors", "short": "1h"}, + Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate1h{slo="monitoring-prometheus-operator-errors"} > (2.8 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate12h{slo="monitoring-prometheus-operator-errors"} > (2.8 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "12h", "slo": "monitoring-prometheus-operator-errors", "short": "1h", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h30m", Expr: intstr.FromString(`prometheus_operator_reconcile_operations:burnrate3h{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99)) and prometheus_operator_reconcile_operations:burnrate2d{slo="monitoring-prometheus-operator-errors"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "2d", "slo": "monitoring-prometheus-operator-errors", "short": "3h"}, + Labels: map[string]string{"severity": "warning", "long": "2d", "slo": "monitoring-prometheus-operator-errors", "short": "3h", "exhaustion": "2w"}, }}, }, }, { @@ -780,22 +780,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`apiserver_request:burnrate3m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (14 * (1-0.99)) and apiserver_request:burnrate30m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (14 * (1-0.99))`), For: "1m", - Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-write-response-errors"}, + Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "1d"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate15m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (7 * (1-0.99)) and apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (7 * (1-0.99))`), + Expr: intstr.FromString(`apiserver_request:burnrate15m{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (5.6 * (1-0.99)) and apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (5.6 * (1-0.99))`), For: "8m", - Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-write-response-errors"}, + Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2d12h"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request:burnrate1h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (2 * (1-0.99)) and apiserver_request:burnrate12h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (2 * (1-0.99))`), + Expr: intstr.FromString(`apiserver_request:burnrate1h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (2.8 * (1-0.99)) and apiserver_request:burnrate12h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (2.8 * (1-0.99))`), For: "30m", - Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-write-response-errors"}, + Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`apiserver_request:burnrate3h{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (1 * (1-0.99)) and apiserver_request:burnrate2d{job="apiserver",slo="apiserver-write-response-errors",verb=~"POST|PUT|PATCH|DELETE"} > (1 * (1-0.99))`), For: "1h30m", - Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-write-response-errors"}, + Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-write-response-errors", "exhaustion": "2w"}, }}, }, }, { @@ -872,22 +872,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99)) and apiserver_request_duration_seconds:burnrate30m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99))`), For: "1m", - Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-read-resource-latency"}, + Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1d"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate15m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99)) and apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99))`), + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate15m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (5.6 * (1-0.99)) and apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (5.6 * (1-0.99))`), For: "8m", - Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-read-resource-latency"}, + Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2d12h"}, }, { Alert: "ErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate1h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99)) and apiserver_request_duration_seconds:burnrate12h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99))`), + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate1h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2.8 * (1-0.99)) and apiserver_request_duration_seconds:burnrate12h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2.8 * (1-0.99))`), For: "30m", - Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-read-resource-latency"}, + Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99)) and apiserver_request_duration_seconds:burnrate2d{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99))`), For: "1h30m", - Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-read-resource-latency"}, + Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2w"}, }}, }, }, { @@ -964,22 +964,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "APIServerLatencyErrorBudgetBurn", Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99)) and apiserver_request_duration_seconds:burnrate30m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (14 * (1-0.99))`), For: "1m", - Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-read-resource-latency"}, + Labels: map[string]string{"severity": "critical", "long": "30m", "short": "3m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "1d"}, }, { Alert: "APIServerLatencyErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate15m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99)) and apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (7 * (1-0.99))`), + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate15m{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (5.6 * (1-0.99)) and apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (5.6 * (1-0.99))`), For: "8m", - Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-read-resource-latency"}, + Labels: map[string]string{"severity": "critical", "long": "3h", "short": "15m", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2d12h"}, }, { Alert: "APIServerLatencyErrorBudgetBurn", - Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate1h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99)) and apiserver_request_duration_seconds:burnrate12h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2 * (1-0.99))`), + Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate1h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2.8 * (1-0.99)) and apiserver_request_duration_seconds:burnrate12h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (2.8 * (1-0.99))`), For: "30m", - Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-read-resource-latency"}, + Labels: map[string]string{"severity": "warning", "long": "12h", "short": "1h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "5d"}, }, { Alert: "APIServerLatencyErrorBudgetBurn", Expr: intstr.FromString(`apiserver_request_duration_seconds:burnrate3h{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99)) and apiserver_request_duration_seconds:burnrate2d{job="apiserver",resource=~"resource|",slo="apiserver-read-resource-latency",verb=~"LIST|GET"} > (1 * (1-0.99))`), For: "1h30m", - Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-read-resource-latency"}, + Labels: map[string]string{"severity": "warning", "long": "2d", "short": "3h", "job": "apiserver", "slo": "apiserver-read-resource-latency", "exhaustion": "2w"}, }}, }, }, { @@ -1020,22 +1020,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "2m", Expr: intstr.FromString(`up:burnrate5m{slo="up-targets"} > (14 * (1-0.99)) and up:burnrate1h{slo="up-targets"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "up-targets"}, + Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "up-targets", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", For: "15m", - Expr: intstr.FromString(`up:burnrate30m{slo="up-targets"} > (7 * (1-0.99)) and up:burnrate6h{slo="up-targets"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "up-targets", "short": "30m"}, + Expr: intstr.FromString(`up:burnrate30m{slo="up-targets"} > (5.6 * (1-0.99)) and up:burnrate6h{slo="up-targets"} > (5.6 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "up-targets", "short": "30m", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h", - Expr: intstr.FromString(`up:burnrate2h{slo="up-targets"} > (2 * (1-0.99)) and up:burnrate1d{slo="up-targets"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "up-targets", "short": "2h"}, + Expr: intstr.FromString(`up:burnrate2h{slo="up-targets"} > (2.8 * (1-0.99)) and up:burnrate1d{slo="up-targets"} > (2.8 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "up-targets", "short": "2h", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", For: "3h", Expr: intstr.FromString(`up:burnrate6h{slo="up-targets"} > (1 * (1-0.99)) and up:burnrate4d{slo="up-targets"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "up-targets", "short": "6h"}, + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "up-targets", "short": "6h", "exhaustion": "4w"}, }}, }, }, { @@ -1076,22 +1076,22 @@ func TestObjective_Burnrates(t *testing.T) { Alert: "ErrorBudgetBurn", For: "2m", Expr: intstr.FromString(`up:burnrate5m{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (14 * (1-0.99)) and up:burnrate1h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (14 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "up-targets"}, + Labels: map[string]string{"severity": "critical", "long": "1h", "short": "5m", "slo": "up-targets", "exhaustion": "2d"}, }, { Alert: "ErrorBudgetBurn", For: "15m", - Expr: intstr.FromString(`up:burnrate30m{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (7 * (1-0.99)) and up:burnrate6h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (7 * (1-0.99))`), - Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "up-targets", "short": "30m"}, + Expr: intstr.FromString(`up:burnrate30m{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (5.6 * (1-0.99)) and up:burnrate6h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (5.6 * (1-0.99))`), + Labels: map[string]string{"severity": "critical", "long": "6h", "slo": "up-targets", "short": "30m", "exhaustion": "5d"}, }, { Alert: "ErrorBudgetBurn", For: "1h", - Expr: intstr.FromString(`up:burnrate2h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (2 * (1-0.99)) and up:burnrate1d{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (2 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "up-targets", "short": "2h"}, + Expr: intstr.FromString(`up:burnrate2h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (2.8 * (1-0.99)) and up:burnrate1d{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (2.8 * (1-0.99))`), + Labels: map[string]string{"severity": "warning", "long": "1d", "slo": "up-targets", "short": "2h", "exhaustion": "10d"}, }, { Alert: "ErrorBudgetBurn", For: "3h", Expr: intstr.FromString(`up:burnrate6h{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (1 * (1-0.99)) and up:burnrate4d{instance!~"(127.0.0.1|localhost).*",slo="up-targets"} > (1 * (1-0.99))`), - Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "up-targets", "short": "6h"}, + Labels: map[string]string{"severity": "warning", "long": "4d", "slo": "up-targets", "short": "6h", "exhaustion": "4w"}, }}, }, }} @@ -1580,39 +1580,217 @@ func TestObjective_IncreaseRules(t *testing.T) { } func Test_windows(t *testing.T) { - ws := Windows(28 * 24 * time.Hour) - - require.Equal(t, Window{ - Severity: critical, - For: 2 * time.Minute, - Long: 1 * time.Hour, - Short: 5 * time.Minute, - Factor: 14, - }, ws[0]) - - require.Equal(t, Window{ - Severity: critical, - For: 15 * time.Minute, - Long: 6 * time.Hour, - Short: 30 * time.Minute, - Factor: 7, - }, ws[1]) + // Test common SLO windows. + // 90d, 84d, 30d, 28d, 14d, 7d + testcases := []struct { + name string + sloWindow time.Duration + expectedWindows []Window + }{{ + name: "A quarter or 90 days", + sloWindow: 90 * 24 * time.Hour, + expectedWindows: []Window{{ + Severity: critical, + For: 6 * time.Minute, + Long: 3*time.Hour + 13*time.Minute, + Short: 16 * time.Minute, + Exhaustion: 6*24*time.Hour + 10*time.Hour + 17*time.Minute, + Factor: 14, + }, { + Severity: critical, + For: 48 * time.Minute, + Long: 19*time.Hour + 17*time.Minute, + Short: time.Hour + 36*time.Minute, + Exhaustion: 16*24*time.Hour + time.Hour + 43*time.Minute, + Factor: 5.6, + }, { + Severity: warning, + For: 3*time.Hour + 13*time.Minute, + Long: ((3*24)+5)*time.Hour + 9*time.Minute, + Short: 6*time.Hour + 26*time.Minute, + Exhaustion: 32*24*time.Hour + 3*time.Hour + 26*time.Minute, + Factor: 2.8, + }, { + Severity: warning, + For: 9*time.Hour + 39*time.Minute, + Long: ((12*24)+20)*time.Hour + 34*time.Minute, + Short: 19*time.Hour + 17*time.Minute, + Exhaustion: 90 * 24 * time.Hour, + Factor: 1, + }}, + }, { + name: "12 weeks or 84 days or a quarter", + sloWindow: 84 * 24 * time.Hour, + expectedWindows: []Window{{ + Severity: critical, + For: 6 * time.Minute, + Long: 3 * time.Hour, + Short: 15 * time.Minute, + Exhaustion: 6 * 24 * time.Hour, + Factor: 14, + }, { + Severity: critical, + For: 45 * time.Minute, + Long: 18 * time.Hour, + Short: 90 * time.Minute, + Exhaustion: 15 * 24 * time.Hour, + Factor: 5.6, + }, { + Severity: warning, + For: 3 * time.Hour, + Long: 3 * 24 * time.Hour, + Short: 6 * time.Hour, + Exhaustion: 30 * 24 * time.Hour, + Factor: 2.8, + }, { + Severity: warning, + For: 9 * time.Hour, + Long: 12 * 24 * time.Hour, + Short: 18 * time.Hour, + Exhaustion: 84 * 24 * time.Hour, + Factor: 1, + }}, + }, { + name: "A month or 30 days", + sloWindow: 30 * 24 * time.Hour, + expectedWindows: []Window{{ + Severity: critical, + For: 2 * time.Minute, + Long: 64 * time.Minute, + Short: 5 * time.Minute, + Exhaustion: 2*24*time.Hour + 3*time.Hour + 26*time.Minute, + Factor: 14, + }, { + Severity: critical, + For: 16 * time.Minute, + Long: 6*time.Hour + 26*time.Minute, + Short: 32 * time.Minute, + Exhaustion: 5*24*time.Hour + 8*time.Hour + 34*time.Minute, + Factor: 5.6, + }, { + Severity: warning, + For: time.Hour + 4*time.Minute, + Long: 25*time.Hour + 43*time.Minute, + Short: 2*time.Hour + 9*time.Minute, + Exhaustion: 10*24*time.Hour + 17*time.Hour + 9*time.Minute, + Factor: 2.8, + }, { + Severity: warning, + For: 3*time.Hour + 13*time.Minute, + Long: ((4*24)+6)*time.Hour + 51*time.Minute, + Short: 6*time.Hour + 26*time.Minute, + Exhaustion: 30 * 24 * time.Hour, + Factor: 1, + }}, + }, { + name: "4 weeks or 28 days", + sloWindow: 28 * 24 * time.Hour, + expectedWindows: []Window{{ + Severity: critical, + For: 2 * time.Minute, + Long: 1 * time.Hour, + Short: 5 * time.Minute, + Exhaustion: 2 * 24 * time.Hour, + Factor: 14, + }, { + Severity: critical, + For: 15 * time.Minute, + Long: 6 * time.Hour, + Short: 30 * time.Minute, + Exhaustion: 5 * 24 * time.Hour, + Factor: 5.6, + }, { + Severity: warning, + For: time.Hour, + Long: 24 * time.Hour, + Short: 2 * time.Hour, + Exhaustion: 10 * 24 * time.Hour, + Factor: 2.8, + }, { + Severity: warning, + For: 3 * time.Hour, + Long: 4 * 24 * time.Hour, + Short: 6 * time.Hour, + Exhaustion: 28 * 24 * time.Hour, + Factor: 1, + }}, + }, { + name: "2 weeks or 14 days", + sloWindow: 14 * 24 * time.Hour, + expectedWindows: []Window{{ + Severity: critical, + For: 1 * time.Minute, + Long: 30 * time.Minute, + Short: 3 * time.Minute, + Exhaustion: 24 * time.Hour, + Factor: 14, + }, { + Severity: critical, + For: 8 * time.Minute, + Long: 3 * time.Hour, + Short: 15 * time.Minute, + Exhaustion: 2*24*time.Hour + 12*time.Hour, + Factor: 5.6, + }, { + Severity: warning, + For: 30 * time.Minute, + Long: 12 * time.Hour, + Short: 1 * time.Hour, + Exhaustion: 5 * 24 * time.Hour, + Factor: 2.8, + }, { + Severity: warning, + For: 90 * time.Minute, + Long: 2 * 24 * time.Hour, + Short: 3 * time.Hour, + Exhaustion: 14 * 24 * time.Hour, + Factor: 1, + }}, + }, { + name: "A week or 7 days", + sloWindow: 7 * 24 * time.Hour, + expectedWindows: []Window{{ + Severity: critical, + For: 1 * time.Minute, + Long: 15 * time.Minute, + Short: 1 * time.Minute, + Exhaustion: 12 * time.Hour, + Factor: 14, + }, { + Severity: critical, + For: 4 * time.Minute, + Long: 90 * time.Minute, + Short: 8 * time.Minute, + Exhaustion: 1*24*time.Hour + 6*time.Hour, + Factor: 5.6, + }, { + Severity: warning, + For: 15 * time.Minute, + Long: 6 * time.Hour, + Short: 30 * time.Minute, + Exhaustion: 2*24*time.Hour + 12*time.Hour, + Factor: 2.8, + }, { + Severity: warning, + For: 45 * time.Minute, + Long: 24 * time.Hour, + Short: 90 * time.Minute, + Exhaustion: 7 * 24 * time.Hour, + Factor: 1, + }}, + }} - require.Equal(t, Window{ - Severity: warning, - For: time.Hour, - Long: 24 * time.Hour, - Short: 2 * time.Hour, - Factor: 2, - }, ws[2]) + require.Len(t, testcases, 6) - require.Equal(t, Window{ - Severity: warning, - For: 3 * time.Hour, - Long: 4 * 24 * time.Hour, - Short: 6 * time.Hour, - Factor: 1, - }, ws[3]) + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + ws := Windows(tc.sloWindow) + require.Len(t, ws, len(tc.expectedWindows)) + for i := range ws { + require.Equal(t, tc.expectedWindows[i], ws[i]) + } + }) + } } func TestObjective_GrafanaRules(t *testing.T) {