kubeapiserver auditloganalyzer: spot handler panics in audit log

Don't let any useragent cause too many panics in apiserver

Author: vrutkovs

pkg/monitortests/kubeapiserver/auditloganalyzer/handle_apiserver_paniced.go

@@ -0,0 +1,44 @@
+package auditloganalyzer
+
+import (
+	"sync"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
+)
+
+type apiserverPaniced struct {
+	lock                    sync.Mutex
+	panicEventsPerUserAgent map[string]sets.Set[types.UID]
+}
+
+func CheckForApiserverPaniced() *apiserverPaniced {
+	return &apiserverPaniced{
+		panicEventsPerUserAgent: map[string]sets.Set[types.UID]{},
+	}
+}
+
+func (s *apiserverPaniced) HandleAuditLogEvent(auditEvent *auditv1.Event, beginning, end *metav1.MicroTime) {
+	if beginning != nil && auditEvent.RequestReceivedTimestamp.Before(beginning) || end != nil && end.Before(&auditEvent.RequestReceivedTimestamp) {
+		return
+	}
+
+	if auditEvent.ResponseStatus == nil {
+		return
+	}
+	if auditEvent.ResponseStatus.Code != 500 {
+		return
+	}
+
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	events, ok := s.panicEventsPerUserAgent[auditEvent.UserAgent]
+	if !ok {
+		events = sets.New[types.UID]()
+	}
+	events.Insert(auditEvent.AuditID)
+	s.panicEventsPerUserAgent[auditEvent.UserAgent] = events
+}

pkg/monitortests/kubeapiserver/auditloganalyzer/handle_excessive_applies.go

@@ -1,11 +1,12 @@
 package auditloganalyzer
 
 import (
+	"strings"
+	"sync"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
-	"strings"
-	"sync"
 )
 
 type excessiveApplies struct {

pkg/monitortests/kubeapiserver/auditloganalyzer/monitortest.go

@@ -19,12 +19,14 @@ type auditLogAnalyzer struct {
 
 	summarizer            *summarizer
 	excessiveApplyChecker *excessiveApplies
+	apiserverPaniced      *apiserverPaniced
 }
 
 func NewAuditLogAnalyzer() monitortestframework.MonitorTest {
 	return &auditLogAnalyzer{
 		summarizer:            NewAuditLogSummarizer(),
 		excessiveApplyChecker: CheckForExcessiveApplies(),
+		apiserverPaniced:      CheckForApiserverPaniced(),
 	}
 }
 
@@ -42,6 +44,7 @@ func (w *auditLogAnalyzer) CollectData(ctx context.Context, storageDir string, b
 	auditLogHandlers := []AuditEventHandler{
 		w.summarizer,
 		w.excessiveApplyChecker,
+		w.apiserverPaniced,
 	}
 	err = GetKubeAuditLogSummary(ctx, kubeClient, &beginning, &end, auditLogHandlers)
 
@@ -122,6 +125,35 @@ func (w *auditLogAnalyzer) EvaluateTestsFromConstructedIntervals(ctx context.Con
 
 	}
 
+	for userAgent, panics := range w.apiserverPaniced.panicEventsPerUserAgent {
+		testName := fmt.Sprintf("user %s must not produce too many apiserver handler panics", userAgent)
+
+		failures := []string{}
+		if len(panics) > 5 {
+			ids := []string{}
+			for _, id := range panics.UnsortedList() {
+				ids = append(ids, string(id))
+			}
+			failures := append(failures, fmt.Sprintf("failed auditIDs: %s", strings.Join(ids, ",")))
+			ret = append(ret,
+				&junitapi.JUnitTestCase{
+					Name: testName,
+					FailureOutput: &junitapi.FailureOutput{
+						Message: strings.Join(failures, "\n"),
+						Output:  "details in audit log",
+					},
+				},
+			)
+		} else {
+			ret = append(ret,
+				&junitapi.JUnitTestCase{
+					Name: testName,
+				},
+			)
+		}
+
+	}
+
 	return ret, nil
 }