feat(loki.secretfilter): Remove redundant secrets_redacted_by_rule_total and secrets_redacted_by_origin metrics (#5970)

The `loki_secretfilter_secrets_redacted_by_category_total` metric (introduced in #5855) partitions by both `rule` and `origin`, making the two single-dimension metrics redundant. Any query with them can be expressed via `sum by (rule|origin) (...)` on the category metric.

Author: kleimkuhler

docs/sources/reference/components/loki/loki.secretfilter.md

@@ -52,7 +52,7 @@ You can use the following arguments with `loki.secretfilter`:
 | `drop_on_timeout`    | `bool`               | When true, drop entries that exceed `processing_timeout` instead of forwarding them unredacted.                             | `false` | no       |
 | `gitleaks_config`    | `string`             | Path to a custom Gitleaks TOML config file. If empty, the default Gitleaks config is used.                                  | `""`    | no       |
 | `label_timed_out`    | `bool`               | When true, adds `secretfilter="timed-out"` to entries forwarded after a processing timeout.                                 | `false` | no       |
-| `origin_label`       | `string`             | Loki label to use as the `origin` dimension in `secrets_redacted_by_origin` and `secrets_redacted_by_category_total`. If empty, `secrets_redacted_by_origin` is not registered and the `origin` label on `secrets_redacted_by_category_total` is set to `""`. | `""`    | no       |
+| `origin_label`       | `string`             | Loki label to use as the `origin` dimension in `secrets_redacted_by_category_total`.                                        | `""`    | no       |
 | `processing_timeout` | `duration`           | Maximum time allowed to process a single log entry. `0` disables the timeout.                                               | `0`     | no       |
 | `rate`               | `float`              | Entry sampling rate in `[0.0, 1.0]` where `1` processes all entries. Unsampled entries are forwarded unchanged.             | `1.0`   | no       |
 | `redact_percent`     | `uint`               | When `redact_with` is not set: percent of the secret to redact (1–100), where 100 is full redaction.                        | `80`    | no       |
@@ -67,7 +67,7 @@ The default configuration may change between {{< param "PRODUCT_NAME" >}} versio
 For consistent behavior, use an external configuration file via `gitleaks_config`.
 {{< /admonition >}}
 
-If you leave `origin_label` empty, the component doesn't register `secrets_redacted_by_origin` and sets the origin label on `secrets_redacted_by_category_total` to `""`.
+If you leave `origin_label` empty, the component sets the origin label on `secrets_redacted_by_category_total` to `""`.
 
 **Redaction behavior:**
 
@@ -84,9 +84,9 @@ Entries that {{< param "PRODUCT_NAME" >}} does not select based on the sampling
 Use a value below `1.0`, for example, `0.1` for 10%, to reduce CPU usage when processing high-volume logs.
 Monitor `loki_secretfilter_entries_bypassed_total` to observe how many entries were skipped.
 
-**Origin metric:** The `origin_label` argument specifies the Loki label the component uses as the origin dimension in the `secrets_redacted_by_origin` and `secrets_redacted_by_category_total` metrics.
+**Origin metric:** The `origin_label` argument specifies the Loki label the component uses as the origin dimension in `secrets_redacted_by_category_total`.
 You can track how many secrets were redacted per source or environment.
-When `origin_label` isn’t set, the component doesn’t register `secrets_redacted_by_origin`, and the `origin` label on `secrets_redacted_by_category_total` defaults to an empty string.
+When `origin_label` isn’t set, the `origin` label on `secrets_redacted_by_category_total` defaults to an empty string.
 
 **Processing timeout:** The `processing_timeout` argument sets a maximum duration for processing each log entry.
 When the timeout is exceeded, the `loki_secretfilter_lines_timed_out_total` metric is incremented.
@@ -133,8 +133,6 @@ The following fields are exported and can be referenced by other components:
 | `loki_secretfilter_lines_timed_out_total`          | Counter | Total number of log lines that exceeded the processing timeout, whether dropped or forwarded.  |
 | `loki_secretfilter_processing_duration_seconds`    | Summary | Time taken to process and redact logs, in seconds.                                             |
 | `loki_secretfilter_secrets_redacted_total`         | Counter | Total number of secrets redacted.                                                              |
-| `loki_secretfilter_secrets_redacted_by_rule_total` | Counter | Number of secrets redacted, partitioned by rule name.                                          |
-| `loki_secretfilter_secrets_redacted_by_origin`     | Counter | Number of secrets redacted, partitioned by origin label, when `origin_label` is set.           |
 | `loki_secretfilter_secrets_redacted_by_category_total` | Counter | Number of secrets redacted, partitioned by rule name and origin label value. The `origin` label is empty when `origin_label` is not set or the label is absent on the entry. |
 
 ## Example

internal/component/loki/secretfilter/secretfilter.go

@@ -120,8 +120,6 @@ type secretDetector interface {
 // Metrics exposed by this component:
 //
 //   - loki_secretfilter_secrets_redacted_total: Total number of secrets that have been redacted.
-//   - loki_secretfilter_secrets_redacted_by_rule_total: Number of secrets redacted, partitioned by rule name.
-//   - loki_secretfilter_secrets_redacted_by_origin: Number of secrets redacted, partitioned by origin label value (only registered when origin_label is set).
 //   - loki_secretfilter_secrets_redacted_by_category_total: Number of secrets redacted, partitioned by rule name and origin label value.
 //   - loki_secretfilter_processing_duration_seconds: Summary of time taken to process and redact log entries.
 //   - loki_secretfilter_entries_bypassed_total: Total number of entries forwarded without processing due to sampling.
@@ -132,12 +130,6 @@ type metrics struct {
 	// Total number of secrets redacted
 	secretsRedactedTotal prometheus.Counter
 
-	// Number of secrets redacted by rule type
-	secretsRedactedByRule *prometheus.CounterVec
-
-	// Number of secrets redacted by specified labels
-	secretsRedactedByOrigin *prometheus.CounterVec
-
 	// Number of secrets redacted by rule and origin (combined)
 	secretsRedactedByCategory *prometheus.CounterVec
 
@@ -155,7 +147,7 @@ type metrics struct {
 }
 
 // newMetrics creates a new set of metrics for the secretfilter component.
-func newMetrics(reg prometheus.Registerer, originLabel string) *metrics {
+func newMetrics(reg prometheus.Registerer) *metrics {
 	var m metrics
 
 	m.secretsRedactedTotal = prometheus.NewCounter(prometheus.CounterOpts{
@@ -164,20 +156,6 @@ func newMetrics(reg prometheus.Registerer, originLabel string) *metrics {
 		Help:      "Total number of secrets that have been redacted.",
 	})
 
-	m.secretsRedactedByRule = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Subsystem: "loki_secretfilter",
-		Name:      "secrets_redacted_by_rule_total",
-		Help:      "Number of secrets redacted, partitioned by rule name.",
-	}, []string{"rule"})
-
-	if originLabel != "" {
-		m.secretsRedactedByOrigin = prometheus.NewCounterVec(prometheus.CounterOpts{
-			Subsystem: "loki_secretfilter",
-			Name:      "secrets_redacted_by_origin",
-			Help:      "Number of secrets redacted, partitioned by origin label value.",
-		}, []string{"origin"})
-	}
-
 	m.secretsRedactedByCategory = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Subsystem: "loki_secretfilter",
 		Name:      "secrets_redacted_by_category_total",
@@ -214,10 +192,6 @@ func newMetrics(reg prometheus.Registerer, originLabel string) *metrics {
 
 	if reg != nil {
 		m.secretsRedactedTotal = util.MustRegisterOrGet(reg, m.secretsRedactedTotal).(prometheus.Counter)
-		m.secretsRedactedByRule = util.MustRegisterOrGet(reg, m.secretsRedactedByRule).(*prometheus.CounterVec)
-		if originLabel != "" {
-			m.secretsRedactedByOrigin = util.MustRegisterOrGet(reg, m.secretsRedactedByOrigin).(*prometheus.CounterVec)
-		}
 		m.secretsRedactedByCategory = util.MustRegisterOrGet(reg, m.secretsRedactedByCategory).(*prometheus.CounterVec)
 		m.processingDuration = util.MustRegisterOrGet(reg, m.processingDuration).(prometheus.Summary)
 		m.entriesBypassedTotal = util.MustRegisterOrGet(reg, m.entriesBypassedTotal).(prometheus.Counter)
@@ -276,7 +250,7 @@ func New(o component.Options, args Arguments) (*Component, error) {
 		log:                o.Logger,
 		receiver:           loki.NewLogsReceiver(loki.WithComponentID(o.ID)),
 		fanout:             loki.NewFanout(args.ForwardTo),
-		metrics:            newMetrics(o.Registerer, args.OriginLabel),
+		metrics:            newMetrics(o.Registerer),
 		debugDataPublisher: debugDataPublisher.(livedebugging.DebugDataPublisher),
 	}
 
@@ -412,12 +386,10 @@ func (c *Component) redactLine(entry loki.Entry, findings []report.Finding) loki
 		line = strings.ReplaceAll(line, originalSecret, replacement)
 
 		c.metrics.secretsRedactedTotal.Inc()
-		c.metrics.secretsRedactedByRule.WithLabelValues(ruleName).Inc()
 		originValue := ""
 		if c.args.OriginLabel != "" && len(entry.Labels) > 0 {
 			if value, ok := entry.Labels[model.LabelName(c.args.OriginLabel)]; ok {
 				originValue = string(value)
-				c.metrics.secretsRedactedByOrigin.WithLabelValues(originValue).Inc()
 			}
 		}
 		c.metrics.secretsRedactedByCategory.WithLabelValues(ruleName, originValue).Inc()
@@ -469,7 +441,7 @@ func (c *Component) Update(args component.Arguments) error {
 	} else {
 		c.sampler.Update(newArgs.Rate)
 	}
-	c.metrics = newMetrics(c.opts.Registerer, newArgs.OriginLabel)
+	c.metrics = newMetrics(c.opts.Registerer)
 
 	level.Debug(c.log).Log(
 		"msg", "loki.secretfilter config updated",

internal/component/loki/secretfilter/secretfilter_test.go

@@ -477,45 +477,6 @@ func TestMetrics(t *testing.T) {
 					"secretsRedactedTotal metric value is incorrect")
 			}
 
-			// Check secretsRedactedByRule - combine all metrics in a single string
-			if len(tc.expectedRedactedByRule) > 0 {
-				var metricStrings strings.Builder
-				metricStrings.WriteString("# HELP loki_secretfilter_secrets_redacted_by_rule_total Number of secrets redacted, partitioned by rule name.\n")
-				metricStrings.WriteString("# TYPE loki_secretfilter_secrets_redacted_by_rule_total counter\n")
-
-				// Add each rule metric
-				for ruleName, expectedCount := range tc.expectedRedactedByRule {
-					metric := fmt.Sprintf(`loki_secretfilter_secrets_redacted_by_rule_total{rule="%s"} %d`,
-						ruleName, expectedCount)
-					metricStrings.WriteString(metric + "\n")
-				}
-
-				// Compare all the metrics at once
-				require.NoError(t,
-					testutil.GatherAndCompare(registry, strings.NewReader(metricStrings.String()),
-						"loki_secretfilter_secrets_redacted_by_rule_total"))
-			}
-
-			// Check secretsRedactedByOrigin when redactions occurred
-			if tc.expectedRedactedTotal > 0 {
-				// Build expected origin label metric
-				var metricStrings strings.Builder
-				metricStrings.WriteString("# HELP loki_secretfilter_secrets_redacted_by_origin Number of secrets redacted, partitioned by origin label value.\n")
-				metricStrings.WriteString("# TYPE loki_secretfilter_secrets_redacted_by_origin counter\n")
-
-				// Add origin label metric
-				if jobValue, exists := labels[model.LabelName("job")]; exists {
-					metric := fmt.Sprintf(`loki_secretfilter_secrets_redacted_by_origin{origin="%s"} %d`,
-						jobValue, tc.expectedRedactedTotal)
-					metricStrings.WriteString(metric + "\n")
-				}
-
-				// Compare the metrics
-				require.NoError(t,
-					testutil.GatherAndCompare(registry, strings.NewReader(metricStrings.String()),
-						"loki_secretfilter_secrets_redacted_by_origin"))
-			}
-
 			// Check secretsRedactedByCategory
 			if len(tc.expectedRedactedByRule) > 0 {
 				jobValue := string(labels[model.LabelName("job")])
@@ -562,8 +523,7 @@ func TestMetrics(t *testing.T) {
 }
 
 // TestMetrics_NoOriginLabel verifies that when origin_label is not set,
-// secrets_redacted_by_category_total still increments (with origin="") but
-// secrets_redacted_by_origin is not registered at all.
+// secrets_redacted_by_category_total still increments with origin="".
 func TestMetrics_NoOriginLabel(t *testing.T) {
 	registry := prometheus.NewRegistry()
 
@@ -590,11 +550,6 @@ func TestMetrics_NoOriginLabel(t *testing.T) {
 	}
 	c.processEntry(context.Background(), entry)
 
-	// secrets_redacted_by_origin must not be registered
-	count, err := testutil.GatherAndCount(registry, "loki_secretfilter_secrets_redacted_by_origin")
-	require.NoError(t, err)
-	require.Equal(t, 0, count, "secrets_redacted_by_origin should not be registered when origin_label is not set")
-
 	// secrets_redacted_by_category_total must increment with origin=""
 	expected := strings.NewReader(
 		"# HELP loki_secretfilter_secrets_redacted_by_category_total Number of secrets redacted, partitioned by rule name and origin label value.\n" +
@@ -628,8 +583,6 @@ func TestMetricsRegistration(t *testing.T) {
 
 	// Increment all metrics to ensure they will be gathered
 	c.metrics.secretsRedactedTotal.Inc()
-	c.metrics.secretsRedactedByRule.WithLabelValues("test_rule").Inc()
-	c.metrics.secretsRedactedByOrigin.WithLabelValues("test_value").Inc()
 	c.metrics.secretsRedactedByCategory.WithLabelValues("test_rule", "test_value").Inc()
 	c.metrics.processingDuration.Observe(0.123)
 
@@ -640,8 +593,6 @@ func TestMetricsRegistration(t *testing.T) {
 	// Create a map of expected metrics
 	expectedMetrics := map[string]bool{
 		"loki_secretfilter_secrets_redacted_total":             false,
-		"loki_secretfilter_secrets_redacted_by_rule_total":     false,
-		"loki_secretfilter_secrets_redacted_by_origin":         false,
 		"loki_secretfilter_secrets_redacted_by_category_total": false,
 		"loki_secretfilter_processing_duration_seconds":        false,
 	}
@@ -719,27 +670,6 @@ func TestMetricsMultipleEntries(t *testing.T) {
 	// We should have 3 redacted secrets (2 grafana-api-key and 1 gcp-api-key)
 	require.Equal(t, float64(3), testutil.ToFloat64(c.metrics.secretsRedactedTotal),
 		"secretsRedactedTotal should count all secrets across multiple entries")
-
-	// Check secretsRedactedByRule for each rule type
-	require.NoError(t,
-		testutil.GatherAndCompare(registry, strings.NewReader(`
-			# HELP loki_secretfilter_secrets_redacted_by_rule_total Number of secrets redacted, partitioned by rule name.
-			# TYPE loki_secretfilter_secrets_redacted_by_rule_total counter
-			loki_secretfilter_secrets_redacted_by_rule_total{rule="grafana-api-key"} 2
-			loki_secretfilter_secrets_redacted_by_rule_total{rule="gcp-api-key"} 1
-		`),
-			"loki_secretfilter_secrets_redacted_by_rule_total"))
-
-	// Check secretsRedactedByOrigin values
-	require.NoError(t,
-		testutil.GatherAndCompare(registry, strings.NewReader(`
-			# HELP loki_secretfilter_secrets_redacted_by_origin Number of secrets redacted, partitioned by origin label value.
-			# TYPE loki_secretfilter_secrets_redacted_by_origin counter
-			loki_secretfilter_secrets_redacted_by_origin{origin="test1"} 1
-			loki_secretfilter_secrets_redacted_by_origin{origin="test2"} 1
-			loki_secretfilter_secrets_redacted_by_origin{origin="test4"} 1
-		`),
-			"loki_secretfilter_secrets_redacted_by_origin"))
 }
 
 // TestArgumentsUpdate validates that the secretfilter component works correctly