Skip to content

Commit 437e51b

Browse files
nyodasnyodas
committed
[endpoint] - Add more info in audit log
In order to increase the visibility we have in the audit logs for some of our most recent endeavors like: - Migrations to FIPS - Clients using Protobuf instead of JSON We need to surface the informations that is related to the specific of the queries. We could add this into metrics but we would lose out on the specifics of who is making those request. Which is the important part when trying to get people to migrate and know the overall state of the platform. This PR aims to rely on the facilities already present in the audit logs. Namely, the custom annotations that are set. We can already see in some case annotation being set on the slow queries >500ms. This would effectively do the same. This patch should be fairly easy to cherry pick in the future as well, if we decide to keep it around once the migration are done. Arguably some of this could be of interest for upstream. Probably the content type more than the TLS cipher. So if we decide to keep it around long term we should create the relevant issue and possibly PR that in. datadog:patch
1 parent 5864a46 commit 437e51b

3 files changed

Lines changed: 178 additions & 0 deletions

File tree

‎staging/src/k8s.io/apiserver/pkg/endpoints/filters/ddog.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
Copyright 2019 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package filters
18+
19+
import (
20+
"crypto/tls"
21+
"k8s.io/apiserver/pkg/audit"
22+
"net/http"
23+
)
24+
25+
// WithDDOGAudits adds additional information about the request to the audit logs.
26+
// This is useful for debugging and troubleshooting.
27+
// TLS Cipher for fips
28+
// Content-Type of the response to track JSON vs protobuf
29+
func WithDDOGAudits(handler http.Handler) http.Handler {
30+
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
31+
ctx := req.Context()
32+
33+
// Set the content type annotation if it is set
34+
if _, ok := w.Header()["Content-Type"]; ok {
35+
audit.AddAuditAnnotation(ctx, "audit.datadoghq.com/contentType", w.Header().Get("Content-Type"))
36+
}
37+
38+
// Set the TLS Cipher annotation if TLS and CipherSuite are set
39+
if req.TLS != nil {
40+
if req.TLS.CipherSuite > 0 {
41+
audit.AddAuditAnnotation(ctx, "audit.datadoghq.com/cipher", tls.CipherSuiteName(req.TLS.CipherSuite))
42+
}
43+
}
44+
45+
handler.ServeHTTP(w, req)
46+
})
47+
}

‎staging/src/k8s.io/apiserver/pkg/endpoints/filters/ddog_test.go‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package filters
2+
3+
import (
4+
"crypto/tls"
5+
"crypto/x509"
6+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
7+
auditinternal "k8s.io/apiserver/pkg/apis/audit"
8+
"k8s.io/apiserver/pkg/audit/policy"
9+
"k8s.io/apiserver/pkg/authentication/user"
10+
"k8s.io/apiserver/pkg/endpoints/request"
11+
"net/http"
12+
"net/http/httptest"
13+
"testing"
14+
)
15+
16+
func TestWithDDOGAudits(t *testing.T) {
17+
handler := func(http.ResponseWriter, *http.Request) {}
18+
shortRunningPath := "/api/v1/namespaces/default/pods/foo"
19+
20+
for _, test := range []struct {
21+
desc string
22+
tlsCipher uint16
23+
contentType string
24+
expected []auditinternal.Event
25+
}{
26+
{
27+
"TLS Cipher set & Content Type set",
28+
tls.TLS_RSA_WITH_RC4_128_SHA,
29+
"application/json",
30+
[]auditinternal.Event{
31+
{
32+
Stage: auditinternal.StageResponseComplete,
33+
Verb: "get",
34+
RequestURI: shortRunningPath,
35+
ResponseStatus: &metav1.Status{Code: 200},
36+
Annotations: map[string]string{
37+
"audit.datadoghq.com/cipher": "TLS_RSA_WITH_RC4_128_SHA",
38+
"audit.datadoghq.com/contentType": "application/json",
39+
},
40+
},
41+
},
42+
}, {
43+
"TLS Cipher set & Content Type unset",
44+
tls.TLS_RSA_WITH_RC4_128_SHA,
45+
"",
46+
[]auditinternal.Event{
47+
{
48+
Stage: auditinternal.StageResponseComplete,
49+
Verb: "get",
50+
RequestURI: shortRunningPath,
51+
ResponseStatus: &metav1.Status{Code: 200},
52+
Annotations: map[string]string{
53+
"audit.datadoghq.com/cipher": "TLS_RSA_WITH_RC4_128_SHA",
54+
},
55+
},
56+
},
57+
}, {
58+
"TLS Cipher unset & Content Type unset",
59+
0,
60+
"",
61+
[]auditinternal.Event{
62+
{
63+
Stage: auditinternal.StageResponseComplete,
64+
Verb: "get",
65+
RequestURI: shortRunningPath,
66+
ResponseStatus: &metav1.Status{Code: 200},
67+
},
68+
},
69+
}, {
70+
"TLS Cipher unset & Content Type set",
71+
0,
72+
"application/json",
73+
[]auditinternal.Event{
74+
{
75+
Stage: auditinternal.StageResponseComplete,
76+
Verb: "get",
77+
RequestURI: shortRunningPath,
78+
ResponseStatus: &metav1.Status{Code: 200},
79+
Annotations: map[string]string{
80+
"audit.datadoghq.com/contentType": "application/json",
81+
},
82+
},
83+
},
84+
},
85+
} {
86+
t.Run(test.desc, func(t *testing.T) {
87+
sink := &fakeAuditSink{}
88+
fakeRuleEvaluator := policy.NewFakePolicyRuleEvaluator(auditinternal.LevelRequestResponse, []auditinternal.Stage{auditinternal.StageRequestReceived, auditinternal.StageResponseStarted})
89+
handler := WithAudit(http.HandlerFunc(handler), sink, fakeRuleEvaluator, func(r *http.Request, ri *request.RequestInfo) bool { return true })
90+
handler = WithAuditInit(handler)
91+
handler = WithDDOGAudits(handler)
92+
93+
req, _ := http.NewRequest("GET", shortRunningPath, nil)
94+
req.RemoteAddr = "127.0.0.1"
95+
req = withTestContext(req, &user.DefaultInfo{Name: "admin"}, nil)
96+
res := httptest.NewRecorder()
97+
98+
if test.tlsCipher > 0 {
99+
req.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{{}}, CipherSuite: test.tlsCipher}
100+
}
101+
if test.contentType != "" {
102+
res.Header().Set("Content-Type", test.contentType)
103+
}
104+
105+
func() {
106+
defer func() {
107+
recover()
108+
}()
109+
handler.ServeHTTP(res, req)
110+
}()
111+
112+
events := sink.Events()
113+
t.Logf("audit log: %v", events)
114+
if len(events) != len(test.expected) {
115+
t.Fatalf("Unexpected amount of lines in audit log: %d", len(events))
116+
}
117+
for i, _ := range test.expected {
118+
event := events[i]
119+
if len(event.Annotations) != len(test.expected[i].Annotations) {
120+
t.Errorf("[%s] expected %d annotations, got %d", test.desc, len(test.expected[i].Annotations), len(event.Annotations))
121+
}
122+
for y, _ := range event.Annotations {
123+
if test.expected[i].Annotations[y] != event.Annotations[y] {
124+
t.Errorf("[%s] expected annotation %s to be %s, got %s", test.desc, y, test.expected[i].Annotations[y], event.Annotations[y])
125+
}
126+
}
127+
}
128+
})
129+
}
130+
}

‎staging/src/k8s.io/apiserver/pkg/server/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,6 +1069,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
10691069
handler = genericapifilters.WithMuxAndDiscoveryComplete(handler, c.lifecycleSignals.MuxAndDiscoveryComplete.Signaled())
10701070
handler = genericfilters.WithPanicRecovery(handler, c.RequestInfoResolver)
10711071
handler = genericapifilters.WithAuditInit(handler)
1072+
handler = genericapifilters.WithDDOGAudits(handler)
10721073
return handler
10731074
}
10741075

0 commit comments

Comments
 (0)