Merge pull request #1208 from cloudflare/rule/report

Add rule/report check

Author: prymitive

cmd/pint/tests/0025_config.txt

@@ -38,7 +38,8 @@ level=INFO msg="Loading configuration file" path=.pint.hcl
       "rule/name",
       "rule/label",
       "rule/link",
-      "rule/reject"
+      "rule/reject",
+      "rule/report"
     ],
     "disabled": [
       "promql/fragile"

cmd/pint/tests/0113_config_env_expand.txt

@@ -43,7 +43,8 @@ level=INFO msg="Loading configuration file" path=.pint.hcl
       "rule/name",
       "rule/label",
       "rule/link",
-      "rule/reject"
+      "rule/reject",
+      "rule/report"
     ]
   },
   "owners": {},

cmd/pint/tests/0202_report_recording_rules.txt

@@ -0,0 +1,33 @@
+! exec pint --no-color lint rules
+! stdout .
+cmp stderr stderr.txt
+
+-- stderr.txt --
+level=INFO msg="Loading configuration file" path=.pint.hcl
+level=INFO msg="Finding all rules to check" paths=["rules"]
+rules/1.yml:3-4 Bug: You cannot add any recording rules to this Prometheus server. (rule/report)
+ 3 | - record: bar
+ 4 |   expr: sum(up)
+
+level=INFO msg="Problems found" Bug=1
+level=ERROR msg="Fatal error" err="found 1 problem(s) with severity Bug or higher"
+-- rules/1.yml --
+- alert: foo
+  expr: up == 0
+- record: bar
+  expr: sum(up)
+
+-- .pint.hcl --
+parser {
+  relaxed = [".*"]
+}
+rule {
+  match {
+    kind = "recording"
+  }
+  report {
+    comment  = "You cannot add any recording rules to this Prometheus server."
+    severity = "bug"
+  }
+}
+

docs/changelog.md

@@ -2,6 +2,10 @@
 
 ## v0.68.0
 
+### Added
+
+- Added [rule/report](checks/rule/report.md) check.
+
 ### Changed
 
 - pint now uses [Prometheus 3.0](https://prometheus.io/blog/2024/11/14/prometheus-3-0/) libraries

docs/checks/rule/report.md

@@ -0,0 +1,81 @@
+---
+layout: default
+parent: Checks
+grand_parent: Documentation
+---
+
+# rule/report
+
+This check will always report a problem for **every matching rule**.
+
+## Configuration
+
+Syntax:
+
+```js
+report {
+  comment  = "..."
+  severity = "bug|warning|info"
+}
+```
+
+- `comment` - set a custom comment that will be added to reported problems.
+- `severity` - set custom severity for reported issues.
+
+## How to enable it
+
+This check is not enabled by default as it requires explicit configuration
+to work.
+To enable it add one or more `rule {...}` blocks that matches some rules and
+then add a `report` block there.
+
+Example where we block all recording rule:
+
+```js
+rule {
+  match {
+    kind = "recording"
+  }
+
+  report {
+    comment  = "You cannot add any recording rules to this Prometheus server."
+    severity = "bug"
+  }
+}
+```
+
+## How to disable it
+
+You can disable this check globally by adding this config block:
+
+```js
+checks {
+  disabled = ["rule/report"]
+}
+```
+
+You can also disable it for all rules inside given file by adding
+a comment anywhere in that file. Example:
+
+```yaml
+# pint file/disable rule/report
+```
+
+Or you can disable it per rule by adding a comment to it. Example:
+
+```yaml
+# pint disable rule/report
+```
+
+## How to snooze it
+
+You can disable this check until given time by adding a comment to it. Example:
+
+```yaml
+# pint snooze $TIMESTAMP rule/report
+```
+
+Where `$TIMESTAMP` is either use [RFC3339](https://www.rfc-editor.org/rfc/rfc3339)
+formatted or `YYYY-MM-DD`.
+Adding this comment will disable `rule/report` _until_ `$TIMESTAMP`, after that
+check will be re-enabled.

internal/checks/base.go

@@ -37,6 +37,7 @@ var (
 		LabelCheckName,
 		RuleLinkCheckName,
 		RejectCheckName,
+		ReportCheckName,
 	}
 	OnlineChecks = []string{
 		AlertsAbsentCheckName,

internal/checks/rule_report.go

@@ -0,0 +1,51 @@
+package checks
+
+import (
+	"context"
+
+	"github.com/cloudflare/pint/internal/discovery"
+	"github.com/cloudflare/pint/internal/parser"
+)
+
+const (
+	ReportCheckName = "rule/report"
+)
+
+func NewReportCheck(c string, s Severity) ReportCheck {
+	return ReportCheck{comment: c, severity: s}
+}
+
+type ReportCheck struct {
+	comment  string
+	severity Severity
+}
+
+func (c ReportCheck) Meta() CheckMeta {
+	return CheckMeta{
+		States: []discovery.ChangeType{
+			discovery.Noop,
+			discovery.Added,
+			discovery.Modified,
+			discovery.Moved,
+		},
+		IsOnline: false,
+	}
+}
+
+func (c ReportCheck) String() string {
+	return ReportCheckName
+}
+
+func (c ReportCheck) Reporter() string {
+	return ReportCheckName
+}
+
+func (c ReportCheck) Check(_ context.Context, _ discovery.Path, rule parser.Rule, _ []discovery.Entry) (problems []Problem) {
+	problems = append(problems, Problem{
+		Lines:    rule.Lines,
+		Reporter: c.Reporter(),
+		Text:     c.comment,
+		Severity: c.severity,
+	})
+	return problems
+}

internal/checks/rule_report_test.go

@@ -0,0 +1,57 @@
+package checks_test
+
+import (
+	"testing"
+
+	"github.com/cloudflare/pint/internal/checks"
+	"github.com/cloudflare/pint/internal/parser"
+	"github.com/cloudflare/pint/internal/promapi"
+)
+
+func TestReportCheck(t *testing.T) {
+	testCases := []checkTest{
+		{
+			description: "report passed problem / warning",
+			content:     "- alert: foo\n  expr: sum(foo)\n  annotations:\n    alert: foo\n",
+			checker: func(_ *promapi.FailoverGroup) checks.RuleChecker {
+				return checks.NewReportCheck("problem reported", checks.Warning)
+			},
+			prometheus: noProm,
+			problems: func(_ string) []checks.Problem {
+				return []checks.Problem{
+					{
+						Lines: parser.LineRange{
+							First: 1,
+							Last:  4,
+						},
+						Reporter: "rule/report",
+						Text:     "problem reported",
+						Severity: checks.Warning,
+					},
+				}
+			},
+		},
+		{
+			description: "report passed problem / info",
+			content:     "- record: foo\n  expr: sum(foo)\n",
+			checker: func(_ *promapi.FailoverGroup) checks.RuleChecker {
+				return checks.NewReportCheck("problem reported", checks.Information)
+			},
+			prometheus: noProm,
+			problems: func(_ string) []checks.Problem {
+				return []checks.Problem{
+					{
+						Lines: parser.LineRange{
+							First: 1,
+							Last:  2,
+						},
+						Reporter: "rule/report",
+						Text:     "problem reported",
+						Severity: checks.Information,
+					},
+				}
+			},
+		},
+	}
+	runTests(t, testCases)
+}