metrics: Add debug metric for non-critical errors

Right now, tetragon_errors_total counts things that aren't really errors, like ProcessMetadataUsernameIgnoredNotInHost. This happens a lot in containers and isn't something we usually need to fix, but it's spamming the error metrics.

This change adds a new tetragon_debug_events_total metric for these kinds of 'casual fails'. I moved ProcessMetadataUsernameIgnoredNotInHost over to this new debug metric so the main error count is actually useful for spotting real problems.

Signed-off-by: Aritra Dey <[email protected]>

Author: AritraDey-Dev

docs/content/en/docs/reference/metrics.md

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Tetragon
+
+package errormetrics
+
+import (
+	"maps"
+	"slices"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cilium/tetragon/pkg/metrics"
+	"github.com/cilium/tetragon/pkg/metrics/consts"
+)
+
+type DebugType int
+
+const (
+	// The username resolution was skipped since the process is not in host
+	// namespaces.
+	ProcessMetadataUsernameIgnoredNotInHost DebugType = iota
+)
+
+var debugTypeLabelValues = map[DebugType]string{
+	ProcessMetadataUsernameIgnoredNotInHost: "process_metadata_username_ignored_not_in_host_namespaces",
+}
+
+func (e DebugType) String() string {
+	return debugTypeLabelValues[e]
+}
+
+var (
+	// Constrained label for debug type
+	debugTypeLabel = metrics.ConstrainedLabel{
+		Name:   "type",
+		Values: slices.Collect(maps.Values(debugTypeLabelValues)),
+	}
+
+	DebugTotal = metrics.MustNewCounter(
+		metrics.NewOpts(
+			consts.MetricsNamespace, "", "debug_events_total",
+			"The total number of Tetragon debug events. For internal use only.",
+			nil, []metrics.ConstrainedLabel{debugTypeLabel}, nil,
+		),
+		nil,
+	)
+)
+
+// Get a new handle on a DebugTotal metric for a DebugType
+func GetDebugTotal(er DebugType) prometheus.Counter {
+	return DebugTotal.WithLabelValues(er.String())
+}
+
+// Increment a DebugTotal for a DebugType
+func DebugTotalInc(er DebugType) {
+	GetDebugTotal(er).Inc()
+}

pkg/metrics/errormetrics/debugmetrics.go

@@ -24,16 +24,12 @@ const (
 	EventFinalizeProcessInfoFailed
 	// Failed to resolve Process uid to username
 	ProcessMetadataUsernameFailed
-	// The username resolution was skipped since the process is not in host
-	// namespaces.
-	ProcessMetadataUsernameIgnoredNotInHost
 )
 
 var errorTypeLabelValues = map[ErrorType]string{
-	ProcessPidTidMismatch:                   "process_pid_tid_mismatch",
-	EventFinalizeProcessInfoFailed:          "event_finalize_process_info_failed",
-	ProcessMetadataUsernameFailed:           "process_metadata_username_failed",
-	ProcessMetadataUsernameIgnoredNotInHost: "process_metadata_username_ignored_not_in_host_namespaces",
+	ProcessPidTidMismatch:          "process_pid_tid_mismatch",
+	EventFinalizeProcessInfoFailed: "event_finalize_process_info_failed",
+	ProcessMetadataUsernameFailed:  "process_metadata_username_failed",
 }
 
 func (e ErrorType) String() string {
@@ -106,6 +102,7 @@ var (
 func RegisterMetrics(group metrics.Group) {
 	group.MustRegister(ErrorTotal)
 	group.MustRegister(HandlerErrors)
+	group.MustRegister(DebugTotal)
 }
 
 func InitMetrics() {
@@ -121,6 +118,10 @@ func InitMetrics() {
 	// NB: We initialize only ops.MSG_OP_UNDEF here, but unknown_opcode can occur for any opcode
 	// that is not explicitly handled.
 	GetHandlerErrors(ops.MSG_OP_UNDEF, HandlePerfUnknownOp).Add(0)
+
+	for er := range debugTypeLabelValues {
+		GetDebugTotal(er).Add(0)
+	}
 }
 
 // Get a new handle on an ErrorTotal metric for an ErrorType

pkg/metrics/errormetrics/errormetrics.go

@@ -42,7 +42,7 @@ func MsgToExecveAccountUnix(unix *processapi.MsgExecveEventUnix) error {
 		}
 
 		if errors.Is(err, ErrNotInHostNs) {
-			errormetrics.ErrorTotalInc(errormetrics.ProcessMetadataUsernameIgnoredNotInHost)
+			errormetrics.DebugTotalInc(errormetrics.ProcessMetadataUsernameIgnoredNotInHost)
 		} else {
 			errormetrics.ErrorTotalInc(errormetrics.ProcessMetadataUsernameFailed)
 		}